Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,7 @@ jobs:
- run: npm run skills:check
- if: matrix.os == 'ubuntu-latest' && matrix.node-version == 22
run: npm run docs:check
- if: matrix.os == 'ubuntu-latest' && matrix.node-version == 22
run: npm run typecheck:hooks
- run: npx tsc --noEmit --project tsconfig.eval.json
- run: npm test
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
},
"scripts": {
"build": "tsdown && node scripts/copy-assets.mjs",
"typecheck:hooks": "tsc --noEmit --project tsconfig.hooks.json",
"dev": "tsx src/index.ts",
"start": "node dist/cli.mjs",
"migrate": "node dist/functions/migrate.js",
Expand Down
6 changes: 3 additions & 3 deletions scripts/docs-sync.state.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
"5000+"
],
"functions": [
"333"
"323"
],
"sourceFiles": [
"268"
"263"
],
"loc": [
"64000"
"62000"
],
"kvScopes": [
"65"
Expand Down
2 changes: 1 addition & 1 deletion scripts/release-gate/graph-index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ function persisted(p, scope) {
}
function graphFiles(p) {
const dir = join(p.data, "state_store.db");
return readdirSync(dir).filter((file) => decodeURIComponent(file).startsWith("mem:graph:")).map((file) => ({ scope: decodeURIComponent(file).replace(/\.bin$/, ""), bytes: statSync(join(dir, file)).size }));
return readdirSync(dir).filter((file) => file.endsWith(".bin") && decodeURIComponent(file).startsWith("mem:graph:")).map((file) => ({ scope: decodeURIComponent(file).replace(/\.bin$/, ""), bytes: statSync(join(dir, file)).size }));
}
function ordinaryReads(p, phase, requestCount = 10) {
const summary = p.traceSummary.get(phase);
Expand Down
34 changes: 27 additions & 7 deletions src/functions/audit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,26 @@ export async function listAuditMonthsDesc(kv: StateKV): Promise<string[]> {
return [...months].sort().reverse();
}

let auditRecordedListener: ((entry: AuditEntry) => void) | null = null;
export type AuditListener = (entry: AuditEntry) => void;

export function setAuditRecordedListener(listener: ((entry: AuditEntry) => void) | null): void {
let auditRecordedListener: AuditListener | null = null;
const auditListeners = new Set<AuditListener>();

export function setAuditRecordedListener(listener: AuditListener | null): void {
auditRecordedListener = listener;
}

export function addAuditListener(listener: AuditListener): () => void {
auditListeners.add(listener);
return () => {
auditListeners.delete(listener);
};
}

function reportListenerFailure(err: unknown): void {
logger.warn("Audit listener failed", { error: err instanceof Error ? err.message : String(err) });
}

export async function recordAudit(
kv: StateKV,
operation: AuditEntry["operation"],
Expand All @@ -174,16 +188,22 @@ export async function recordAudit(
operation,
userId,
functionId,
targetIds,
details,
targetIds: Object.freeze([...targetIds]) as string[],
details: Object.freeze({ ...details }),
qualityScore,
};
Object.freeze(entry);
const month = auditMonthOf(entry.timestamp);
await kv.set(KV.auditMonth(month), entry.id, entry);
await markAuditMonth(kv, month);
try {
auditRecordedListener?.(entry);
} catch {}
const listeners = auditRecordedListener ? [auditRecordedListener, ...auditListeners] : [...auditListeners];
for (const listener of listeners) {
try {
void Promise.resolve(listener(entry)).catch(reportListenerFailure);
} catch (err) {
reportListenerFailure(err);
}
}
return entry;
}

Expand Down
141 changes: 113 additions & 28 deletions src/functions/flow-compress.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,24 @@
import type { IIIClient } from "iii-sdk";
import type { StateKV } from "../state/kv.js";
import { collectAll, type StateKV } from "../state/kv.js";
import { KV, generateId } from "../state/schema.js";
import type { Action, ActionEdge, RoutineRun, MemoryProvider } from "../types.js";
import { recordAudit } from "./audit.js";
import { withKeyedLock } from "../state/keyed-mutex.js";
import { sharesMemoryScope, supersededAt } from "../state/memory-utils.js";
import type { Action, ActionEdge, Memory, RoutineRun, MemoryProvider } from "../types.js";
import { getAgentId } from "../config.js";
import { logger } from "../logger.js";
import { safeAudit } from "./audit.js";
import { indexMemory, unindexMemory } from "./search.js";
import { memoryTitleFromContent } from "./remember.js";

type FlowMemory = Memory & { metadata?: { flowCompressed?: boolean; actionCount?: number; actionIds?: string[] } };

function trimmed(value: unknown, max = 512): string | undefined {
return typeof value === "string" && value.trim() ? value.trim().slice(0, max) : undefined;
}

function flowKey(ids: string[]): string {
return [...new Set(ids)].sort().join("\n");
}

const FLOW_COMPRESS_SYSTEM = `You are a workflow summarizer. Given a completed action chain, produce a concise summary capturing:
1. The overall goal and outcome
Expand All @@ -25,7 +41,19 @@ export function registerFlowCompressFunction(
provider: MemoryProvider,
): void {
sdk.registerFunction("mem::flow-compress",
async (data: { runId?: string; actionIds?: string[]; project?: string }) => {
async (data: { runId?: string; actionIds?: string[]; project?: string; agentId?: string }) => {
if (data.runId !== undefined && typeof data.runId !== "string") {
return { success: false, error: "runId must be a string" };
}
if (
data.actionIds !== undefined &&
(!Array.isArray(data.actionIds) || data.actionIds.some((id) => typeof id !== "string"))
) {
return { success: false, error: "actionIds must be an array of strings" };
}
const requestedProject = trimmed(data.project);
const agentId = trimmed(data.agentId, 128) ?? getAgentId();
data = { ...data, project: requestedProject };
let actionsToCompress: Action[] = [];

if (data.runId) {
Expand Down Expand Up @@ -81,40 +109,97 @@ export function registerFlowCompressFunction(
prompt,
);
const summary = parseFlowSummary(response);
const ts = new Date().toISOString();

const memory = {
id: generateId("mem"),
createdAt: ts,
updatedAt: ts,
type: "workflow" as const,
title: summary.goal || `Workflow: ${doneActions.length} actions`,
content: formatSummary(summary),
concepts: extractConcepts(doneActions),
files: extractFiles(doneActions),
sessionIds: [],
strength: 1.0,
version: 1,
isLatest: true,
metadata: {
flowCompressed: true,
const content = formatSummary(summary);
if (!content) {
logger.warn("Flow summary had no recognized sections; nothing stored", {
actionCount: doneActions.length,
actionIds: doneActions.map((a) => a.id),
},
};
});
return {
success: false,
error: "summary could not be parsed",
compressed: 0,
};
}

const actionProjects = new Set(doneActions.map((a) => a.project));
const project = requestedProject
? doneActions.every((a) => a.project === requestedProject) ? requestedProject : undefined
: actionProjects.size === 1 ? doneActions[0].project : undefined;
const actionIds = doneActions.map((a) => a.id);
const key = flowKey(actionIds);

await kv.set(KV.memories, memory.id, memory);
await recordAudit(kv, "compress", "mem::flow-compress", [memory.id], {
const { memory, previous } = await withKeyedLock("mem:remember", async () => {
const ts = new Date().toISOString();
const scope = { project, agentId };
const memories = await collectAll<FlowMemory>(kv, KV.memories);
const previous = memories.find((m) =>
m.isLatest !== false &&
m.metadata?.flowCompressed === true &&
Array.isArray(m.metadata.actionIds) &&
flowKey(m.metadata.actionIds) === key &&
sharesMemoryScope(m, scope) &&
(m.project || undefined) === project,
);
const memory: FlowMemory = {
id: generateId("mem"),
createdAt: ts,
updatedAt: ts,
type: "workflow",
title: summary.goal ? memoryTitleFromContent(summary.goal) : `Workflow: ${doneActions.length} actions`,
content,
concepts: extractConcepts(doneActions),
files: extractFiles(doneActions),
sessionIds: [],
strength: 1.0,
version: previous ? (previous.version ?? 1) + 1 : 1,
isLatest: true,
...(previous && { parentId: previous.id, supersedes: [previous.id, ...(previous.supersedes ?? [])] }),
...(agentId && { agentId }),
...(project && { project }),
metadata: {
flowCompressed: true,
actionCount: doneActions.length,
actionIds,
},
};
await kv.set(KV.memories, memory.id, memory);
if (previous) {
try {
await kv.set(KV.memories, previous.id, {
...previous,
isLatest: false,
updatedAt: supersededAt(previous.updatedAt, ts),
});
} catch (err) {
await kv.delete(KV.memories, memory.id);
throw err;
}
unindexMemory(previous.id);
}
return { memory, previous };
});

await safeAudit(kv, previous ? "evolve" : "compress", "mem::flow-compress", [memory.id], {
action: "compress_flow",
flowCompressed: true,
actionCount: doneActions.length,
project: data.project,
...(previous && { oldId: previous.id, newId: memory.id }),
project,
});
try {
await indexMemory(kv, memory);
} catch (err) {
logger.warn("Flow-compressed memory indexing failed", {
memId: memory.id,
error: err instanceof Error ? err.message : String(err),
});
}

return {
success: true,
compressed: doneActions.length,
memoryId: memory.id,
...(previous && { supersededId: previous.id }),
summary,
};
} catch (err) {
Expand Down Expand Up @@ -166,7 +251,7 @@ function parseFlowSummary(response: string): {
} {
const extract = (tag: string): string => {
const match = response.match(
new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`),
new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)</${tag}>`),
);
return match ? match[1].trim() : "";
};
Expand Down
8 changes: 6 additions & 2 deletions src/functions/observe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,12 @@ export function registerObserveFunction(
const promptRef = durable || typeof payload.eventId !== "string" ? obsId : payload.eventId;

return withKeyedLock(`obs:${payload.sessionId}`, async () => {
const existing = await kv.list<CompressedObservation>(KV.observations(payload.sessionId));
const stored = durable ? existing.find((o) => o?.id === obsId) : undefined;
const direct = durable
? await kv.get<CompressedObservation>(KV.observations(payload.sessionId), obsId).catch(() => null)
: null;
const keyed = direct?.id === obsId ? direct : undefined;
const existing = keyed ? [] : await kv.list<CompressedObservation>(KV.observations(payload.sessionId));
const stored = keyed ?? (durable ? existing.find((o) => o?.id === obsId) : undefined);
if (stored) {
if (livePrompt) await recordLivePrompt(kv, payload.sessionId, livePrompt, promptRef);
await restoreIndexEntries(stored);
Expand Down
45 changes: 33 additions & 12 deletions src/functions/reflect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { createGraphIndexReader } from "../state/graph-retrieval-index.js";
import { lookupObservationSession } from "../state/obs-index.js";
import type { IIIClient } from "iii-sdk";
import type { StateKV } from "../state/kv.js";
import { collectAll } from "../state/kv.js";
import { collectAll, compareInsertionOrder, insertionOrderKey } from "../state/kv.js";
import { KV, fingerprintId } from "../state/schema.js";
import type {
Insight,
Expand Down Expand Up @@ -570,19 +570,40 @@ export function registerReflectFunctions(
if (limit === null) return invalidOption("limit must be a positive safe integer");
if (!validConfidence(data?.minConfidence)) return invalidOption("minConfidence must be a finite number between 0 and 1");
const minConfidence = data?.minConfidence ?? 0;
let items = await collectAll<Insight>(kv, KV.insights);

items = items.filter(
(i) => !i.deleted && i.confidence >= minConfidence,
);

if (data?.project) {
items = items.filter((i) => i.project === data.project);
const paged = typeof kv.iterate === "function";
const redisOrder = paged && kv.backend === "redis";
type Entry = { insight: Insight; order: ReturnType<typeof insertionOrderKey> };
const items: Entry[] = [];
const compare = (a: Entry, b: Entry) =>
b.insight.confidence - a.insight.confidence ||
(redisOrder ? compareInsertionOrder(a.order, b.order) : 0);
const consider = (insight: Insight) => {
if (insight.deleted || !(insight.confidence >= minConfidence)) return;
if (data?.project && insight.project !== data.project) return;
const entry: Entry = {
insight,
order: redisOrder ? insertionOrderKey(insight) : { at: 0, id: "" },
};
if (items.length === limit && compare(entry, items[items.length - 1]!) >= 0) return;
let low = 0;
let high = items.length;
while (low < high) {
const middle = (low + high) >>> 1;
if (compare(entry, items[middle]!) < 0) high = middle;
else low = middle + 1;
}
items.splice(low, 0, entry);
if (items.length > limit) items.pop();
};
if (paged) {
for await (const page of kv.iterate<Insight>(KV.insights)) {
for (const insight of page) consider(insight);
}
} else {
for (const insight of await collectAll<Insight>(kv, KV.insights)) consider(insight);
}

items.sort((a, b) => b.confidence - a.confidence);

const result = { success: true, insights: items.slice(0, limit) };
const result = { success: true, insights: items.map((entry) => entry.insight) };
return checkPayloadFrameSize(result, "reduce limit or filter by project") ?? result;
},
);
Expand Down
Loading
Loading