diff --git a/src/functions/reflect.ts b/src/functions/reflect.ts index 7997c17137..7dbd2ff6fd 100644 --- a/src/functions/reflect.ts +++ b/src/functions/reflect.ts @@ -2,7 +2,7 @@ import { createGraphIndexReader } from "../state/graph-retrieval-index.js"; import { lookupObservationSession } from "../state/obs-index.js"; import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../state/kv.js"; -import { collectAll } from "../state/kv.js"; +import { collectAll, compareInsertionOrder, insertionOrderKey } from "../state/kv.js"; import { KV, fingerprintId } from "../state/schema.js"; import type { Insight, @@ -570,19 +570,40 @@ export function registerReflectFunctions( if (limit === null) return invalidOption("limit must be a positive safe integer"); if (!validConfidence(data?.minConfidence)) return invalidOption("minConfidence must be a finite number between 0 and 1"); const minConfidence = data?.minConfidence ?? 0; - let items = await collectAll(kv, KV.insights); - - items = items.filter( - (i) => !i.deleted && i.confidence >= minConfidence, - ); - - if (data?.project) { - items = items.filter((i) => i.project === data.project); + const paged = typeof kv.iterate === "function"; + const redisOrder = paged && kv.backend === "redis"; + type Entry = { insight: Insight; order: ReturnType }; + const items: Entry[] = []; + const compare = (a: Entry, b: Entry) => + b.insight.confidence - a.insight.confidence || + (redisOrder ? compareInsertionOrder(a.order, b.order) : 0); + const consider = (insight: Insight) => { + if (insight.deleted || !(insight.confidence >= minConfidence)) return; + if (data?.project && insight.project !== data.project) return; + const entry: Entry = { + insight, + order: redisOrder ? insertionOrderKey(insight) : { at: 0, id: "" }, + }; + if (items.length === limit && compare(entry, items[items.length - 1]!) >= 0) return; + let low = 0; + let high = items.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compare(entry, items[middle]!) < 0) high = middle; + else low = middle + 1; + } + items.splice(low, 0, entry); + if (items.length > limit) items.pop(); + }; + if (paged) { + for await (const page of kv.iterate(KV.insights)) { + for (const insight of page) consider(insight); + } + } else { + for (const insight of await collectAll(kv, KV.insights)) consider(insight); } - items.sort((a, b) => b.confidence - a.confidence); - - const result = { success: true, insights: items.slice(0, limit) }; + const result = { success: true, insights: items.map((entry) => entry.insight) }; return checkPayloadFrameSize(result, "reduce limit or filter by project") ?? result; }, ); diff --git a/src/state/kv.ts b/src/state/kv.ts index 8d08c58af3..5aa5bd8a5f 100644 --- a/src/state/kv.ts +++ b/src/state/kv.ts @@ -94,14 +94,23 @@ function idKey(value: unknown): string { return typeof id === 'string' ? id : typeof id === 'number' ? String(id) : '' } +export function insertionOrderKey(value: unknown): { at: number; id: string } { + return { at: orderKey(value), id: idKey(value) } +} + +export function compareInsertionOrder( + a: ReturnType, + b: ReturnType, +): number { + if (a.at !== b.at) return a.at < b.at ? -1 : 1 + if (a.id === b.id) return 0 + return a.id < b.id ? -1 : 1 +} + export function orderLikeInsertion(values: T[]): T[] { return values - .map((value) => ({ value, at: orderKey(value), id: idKey(value) })) - .sort((a, b) => { - if (a.at !== b.at) return a.at < b.at ? -1 : 1 - if (a.id === b.id) return 0 - return a.id < b.id ? -1 : 1 - }) + .map((value) => ({ value, ...insertionOrderKey(value) })) + .sort(compareInsertionOrder) .map((entry) => entry.value) } diff --git a/test/fixtures/insight-list-memory.ts b/test/fixtures/insight-list-memory.ts new file mode 100644 index 0000000000..20ca4221e2 --- /dev/null +++ b/test/fixtures/insight-list-memory.ts @@ -0,0 +1,94 @@ +import { strict as assert } from "node:assert"; +import { createHash } from "node:crypto"; +import { registerReflectFunctions } from "../../src/functions/reflect.js"; +import { StateKV } from "../../src/state/kv.js"; +import { KV } from "../../src/state/schema.js"; + +const backend = process.argv[2]; +assert(backend === "file" || backend === "redis"); +assert.equal(typeof global.gc, "function", "Run this probe with --expose-gc"); + +const count = 4096; +const functions = new Map unknown>(); +let reads = 0; +let peakHeapBytes = 0; +let modelRequests = 0; + +function sampleHeap() { + global.gc!(); + peakHeapBytes = Math.max(peakHeapBytes, process.memoryUsage().heapUsed); +} + +function record(index: number) { + const bytes = Buffer.alloc(12 * 1024, index % 251); + bytes.writeUInt32LE(index); + const concept = bytes.toString("base64"); + return JSON.parse(JSON.stringify({ + id: `historical-${index.toString().padStart(5, "0")}`, + title: `Historical insight ${index}`, + content: `Complete insight ${index}; Unicode Ж and é remain intact.`, + confidence: (index + 1) / count, + reinforcements: 3, + sourceConceptCluster: [concept], + sourceMemoryIds: [`memory-${index}`], + sourceLessonIds: [], + sourceCrystalIds: [], + tags: [concept], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + decayRate: 0.05, + project: "large-store", + legacy_extra: { index, values: [0, false, null] }, + })); +} + +const sdk = { + registerFunction: (id: string | { id: string }, handler: (data: unknown) => unknown) => { + functions.set(typeof id === "string" ? id : id.id, handler); + }, + trigger: async ({ function_id, payload }: { + function_id: string; + payload: { scope: string; key?: string }; + }) => { + assert.equal(payload.scope, KV.insights); + if (function_id === "state::list_keys") { + return Array.from({ length: count }, (_, index) => String(index)); + } + assert.equal(function_id, "state::get"); + const index = Number(payload.key); + assert(Number.isInteger(index) && index >= 0 && index < count); + // Sample while the handler still owns earlier pages, before the next read. + if (reads % 200 === 0) sampleHeap(); + reads++; + return record(index); + }, +}; + +const unexpectedModel = async () => { + modelRequests++; + throw new Error("Listing insights must not request a model"); +}; +const kv = new StateKV(sdk as never, { backend }); +registerReflectFunctions(sdk as never, kv, { + name: "no-model", + compress: unexpectedModel, + summarize: unexpectedModel, +} as never); + +sampleHeap(); +const baselineHeapBytes = peakHeapBytes; +const result = await functions.get("mem::insight-list")!({ limit: 1, project: "large-store" }); +sampleHeap(); +assert.deepEqual(result, { success: true, insights: [record(count - 1)] }); +assert.equal(reads, count); +assert.equal(modelRequests, 0); +console.log(JSON.stringify({ + backend, + records: count, + reads, + modelRequests, + baselineHeapBytes, + peakHeapBytes, + heapGrowthBytes: peakHeapBytes - baselineHeapBytes, + completeResultSha256: createHash("sha256").update(JSON.stringify(result)).digest("hex"), +})); diff --git a/test/fixtures/insight-list.json b/test/fixtures/insight-list.json new file mode 100644 index 0000000000..ae737c2fbd --- /dev/null +++ b/test/fixtures/insight-list.json @@ -0,0 +1,1061 @@ +{ + "records": { + "late": { + "id": "ins_date_late", + "title": "Fixture late", + "content": "Complete historical insight late; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-04T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "late", + "values": [ + 0, + false, + null + ] + } + }, + "id_late": { + "id": "ins_mjxjdhc0_a", + "title": "Fixture id_late", + "content": "Complete historical insight id_late; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "id_late", + "values": [ + 0, + false, + null + ] + } + }, + "high_b": { + "id": "high-b", + "title": "Fixture high_b", + "content": "Complete historical insight high_b; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.95, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "B", + "legacy_extra": { + "keep": "high_b", + "values": [ + 0, + false, + null + ] + } + }, + "field_early": { + "id": "ins_field_early", + "title": "Fixture field_early", + "content": "Complete historical insight field_early; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "field_early", + "values": [ + 0, + false, + null + ] + } + }, + "tie_b": { + "id": "plain-b", + "title": "Fixture tie_b", + "content": "Complete historical insight tie_b; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "tie_b", + "values": [ + 0, + false, + null + ] + } + }, + "deleted_a": { + "id": "deleted-a", + "title": "Fixture deleted_a", + "content": "Complete historical insight deleted_a; commas, accents é and Unicode Ж remain intact.", + "confidence": 1, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "deleted_a", + "values": [ + 0, + false, + null + ] + }, + "deleted": true + }, + "no_time": { + "id": "plain-z", + "title": "Fixture no_time", + "content": "Complete historical insight no_time; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "not-a-date", + "updatedAt": "not-a-date", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "no_time", + "values": [ + 0, + false, + null + ] + } + }, + "tie_a": { + "id": "plain-a", + "title": "Fixture tie_a", + "content": "Complete historical insight tie_a; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "tie_a", + "values": [ + 0, + false, + null + ] + } + }, + "unscoped": { + "id": "unscoped", + "title": "Fixture unscoped", + "content": "Complete historical insight unscoped; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.9, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "legacy_extra": { + "keep": "unscoped", + "values": [ + 0, + false, + null + ] + } + }, + "id_early": { + "id": "ins_mjt91xc0_z", + "title": "Fixture id_early", + "content": "Complete historical insight id_early; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-05T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "id_early", + "values": [ + 0, + false, + null + ] + } + }, + "low_a": { + "id": "low-a", + "title": "Fixture low_a", + "content": "Complete historical insight low_a; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.7, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "low_a", + "values": [ + 0, + false, + null + ] + } + }, + "fallback": { + "id": "fallback", + "title": "Fixture fallback", + "content": "Complete historical insight fallback; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.8, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "not-a-date", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "fallback", + "values": [ + 0, + false, + null + ] + }, + "timestamp": "2026-01-01T00:00:00.000Z", + "startedAt": "2026-01-05T00:00:00.000Z" + }, + "top_a": { + "id": "top-a", + "title": "Fixture top_a", + "content": "Complete historical insight top_a; commas, accents é and Unicode Ж remain intact.", + "confidence": 1, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "top_a", + "values": [ + 0, + false, + null + ] + } + }, + "zero_a": { + "id": "zero-a", + "title": "Fixture zero_a", + "content": "Complete historical insight zero_a; commas, accents é and Unicode Ж remain intact.", + "confidence": 0, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "A", + "legacy_extra": { + "keep": "zero_a", + "values": [ + 0, + false, + null + ] + } + }, + "low_b": { + "id": "low-b", + "title": "Fixture low_b", + "content": "Complete historical insight low_b; commas, accents é and Unicode Ж remain intact.", + "confidence": 0.6, + "reinforcements": 3, + "sourceConceptCluster": [ + "alpha", + "comma,inside", + "Ж" + ], + "sourceMemoryIds": [ + "mem-one", + "mem-two" + ], + "sourceLessonIds": [ + "lesson-one" + ], + "sourceCrystalIds": [ + "crystal-one" + ], + "evidenceVersion": 1, + "tags": [ + "alpha", + "same content preserved" + ], + "createdAt": "2026-01-02T00:00:00.000Z", + "updatedAt": "2026-01-09T00:00:00.000Z", + "lastReinforcedAt": "2026-01-08T00:00:00.000Z", + "lastDecayedAt": "2026-01-07T00:00:00.000Z", + "decayRate": 0.05, + "project": "B", + "legacy_extra": { + "keep": "low_b", + "values": [ + 0, + false, + null + ] + } + } + }, + "pages": [ + [ + "late", + "id_late", + "high_b" + ], + [ + "field_early", + "tie_b", + "deleted_a", + "no_time" + ], + [ + "tie_a", + "unscoped", + "id_early", + "low_a" + ], + [ + "fallback", + "top_a", + "zero_a", + "low_b" + ] + ], + "cases": [ + { + "name": "small3_projectA_filters_before_selection", + "request": { + "project": "A", + "limit": 3 + }, + "expected": { + "file": [ + "top_a", + "late", + "id_late" + ], + "redis": [ + "top_a", + "id_early", + "fallback" + ] + } + }, + { + "name": "small5_projectA_confidence_ties", + "request": { + "project": "A", + "limit": 5 + }, + "expected": { + "file": [ + "top_a", + "late", + "id_late", + "field_early", + "tie_b" + ], + "redis": [ + "top_a", + "id_early", + "fallback", + "field_early", + "tie_a" + ] + } + }, + { + "name": "small5_global", + "request": { + "limit": 5 + }, + "expected": { + "file": [ + "top_a", + "high_b", + "unscoped", + "late", + "id_late" + ], + "redis": [ + "top_a", + "high_b", + "unscoped", + "id_early", + "fallback" + ] + } + }, + { + "name": "default_global_all14", + "request": {}, + "expected": { + "file": [ + "top_a", + "high_b", + "unscoped", + "late", + "id_late", + "field_early", + "tie_b", + "no_time", + "tie_a", + "id_early", + "fallback", + "low_a", + "low_b", + "zero_a" + ], + "redis": [ + "top_a", + "high_b", + "unscoped", + "id_early", + "fallback", + "field_early", + "tie_a", + "tie_b", + "id_late", + "late", + "no_time", + "low_a", + "low_b", + "zero_a" + ] + } + }, + { + "name": "empty_project_is_unfiltered", + "request": { + "project": "" + }, + "expected": { + "file": [ + "top_a", + "high_b", + "unscoped", + "late", + "id_late", + "field_early", + "tie_b", + "no_time", + "tie_a", + "id_early", + "fallback", + "low_a", + "low_b", + "zero_a" + ], + "redis": [ + "top_a", + "high_b", + "unscoped", + "id_early", + "fallback", + "field_early", + "tie_a", + "tie_b", + "id_late", + "late", + "no_time", + "low_a", + "low_b", + "zero_a" + ] + } + }, + { + "name": "project_filter_exact_no_trim", + "request": { + "project": " A " + }, + "expected": { + "file": [], + "redis": [] + } + }, + { + "name": "projectB", + "request": { + "project": "B" + }, + "expected": { + "file": [ + "high_b", + "low_b" + ], + "redis": [ + "high_b", + "low_b" + ] + } + }, + { + "name": "projectA_minConfidence_inclusive", + "request": { + "project": "A", + "minConfidence": 0.8 + }, + "expected": { + "file": [ + "top_a", + "late", + "id_late", + "field_early", + "tie_b", + "no_time", + "tie_a", + "id_early", + "fallback" + ], + "redis": [ + "top_a", + "id_early", + "fallback", + "field_early", + "tie_a", + "tie_b", + "id_late", + "late", + "no_time" + ] + } + }, + { + "name": "projectA_confidence1", + "request": { + "project": "A", + "minConfidence": 1 + }, + "expected": { + "file": [ + "top_a" + ], + "redis": [ + "top_a" + ] + } + }, + { + "name": "projectA_limit501_caps500", + "request": { + "project": "A", + "limit": 501 + }, + "expected": { + "file": [ + "top_a", + "late", + "id_late", + "field_early", + "tie_b", + "no_time", + "tie_a", + "id_early", + "fallback", + "low_a", + "zero_a" + ], + "redis": [ + "top_a", + "id_early", + "fallback", + "field_early", + "tie_a", + "tie_b", + "id_late", + "late", + "no_time", + "low_a", + "zero_a" + ] + } + }, + { + "name": "limit_MAX_SAFE_INTEGER_caps500", + "request": { + "limit": 9007199254740991 + }, + "expected": { + "file": [ + "top_a", + "high_b", + "unscoped", + "late", + "id_late", + "field_early", + "tie_b", + "no_time", + "tie_a", + "id_early", + "fallback", + "low_a", + "low_b", + "zero_a" + ], + "redis": [ + "top_a", + "high_b", + "unscoped", + "id_early", + "fallback", + "field_early", + "tie_a", + "tie_b", + "id_late", + "late", + "no_time", + "low_a", + "low_b", + "zero_a" + ] + } + }, + { + "name": "unknown_project_empty", + "request": { + "project": "missing", + "limit": 1 + }, + "expected": { + "file": [], + "redis": [] + } + } + ], + "invalid": [ + { + "request": { + "limit": 0 + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + }, + { + "request": { + "limit": -1 + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + }, + { + "request": { + "limit": 1.5 + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + }, + { + "request": { + "limit": "3" + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + }, + { + "request": { + "limit": null + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + }, + { + "request": { + "limit": 9007199254740992 + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + }, + { + "request": { + "minConfidence": -0.1 + }, + "expected": { + "success": false, + "invalid": true, + "error": "minConfidence must be a finite number between 0 and 1" + } + }, + { + "request": { + "minConfidence": 1.1 + }, + "expected": { + "success": false, + "invalid": true, + "error": "minConfidence must be a finite number between 0 and 1" + } + }, + { + "request": { + "minConfidence": "0.8" + }, + "expected": { + "success": false, + "invalid": true, + "error": "minConfidence must be a finite number between 0 and 1" + } + }, + { + "request": { + "minConfidence": null + }, + "expected": { + "success": false, + "invalid": true, + "error": "minConfidence must be a finite number between 0 and 1" + } + }, + { + "request": { + "limit": 0, + "minConfidence": "bad" + }, + "expected": { + "success": false, + "invalid": true, + "error": "limit must be a positive safe integer" + } + } + ] +} diff --git a/test/insight-list.test.ts b/test/insight-list.test.ts new file mode 100644 index 0000000000..30a890ac6f --- /dev/null +++ b/test/insight-list.test.ts @@ -0,0 +1,106 @@ +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { registerReflectFunctions } from "../src/functions/reflect.js"; +import { StateKV } from "../src/state/kv.js"; +import { KV } from "../src/state/schema.js"; + +type Backend = "file" | "redis"; +type Row = Record; +const fixture = JSON.parse(readFileSync(new URL("./fixtures/insight-list.json", import.meta.url), "utf8")) as { + records: Record; + pages: string[][]; + cases: { name: string; request: Row; expected: Record }[]; + invalid: { request: Row; expected: Row }[]; +}; +const rows = fixture.pages.flat().map((alias) => fixture.records[alias]!); + +function register(kv: unknown) { + const functions = new Map Promise>(); + const sdk = { + registerFunction: (id: string | { id: string }, handler: (data: unknown) => Promise) => { + functions.set(typeof id === "string" ? id : id.id, handler); + }, + }; + registerReflectFunctions(sdk as never, kv as never, {} as never); + return (data: unknown) => functions.get("mem::insight-list")!(data); +} + +function paged(backend: Backend) { + let pagesRead = 0; + const kv = { + backend, + async *iterate(scope: string) { + expect(scope).toBe(KV.insights); + for (const page of fixture.pages) { + pagesRead++; + yield page.map((alias) => structuredClone(fixture.records[alias]!)); + } + }, + }; + return { call: register(kv), pagesRead: () => pagesRead }; +} + +describe.each(["file", "redis"] as const)("complete historical insight list (%s)", (backend) => { + it.each(fixture.cases)("preserves exact records and order: $name", async ({ request, expected }) => { + const f = paged(backend); + expect(await f.call(request)).toEqual({ + success: true, + insights: expected[backend].map((alias) => fixture.records[alias]), + }); + expect(f.pagesRead()).toBe(fixture.pages.length); + }); + + it.each(fixture.invalid)("validates before reading: $request", async ({ request, expected }) => { + const f = paged(backend); + expect(await f.call(request)).toEqual(expected); + expect(f.pagesRead()).toBe(0); + }); + + it("preserves complete order through real StateKV page reads", async () => { + const sdk = { + trigger: async ({ function_id, payload }: { function_id: string; payload: { scope: string; key?: string } }) => { + expect(payload.scope).toBe(KV.insights); + if (function_id === "state::list_keys") return rows.map((_, index) => String(index)); + expect(function_id).toBe("state::get"); + return structuredClone(rows[Number(payload.key)]); + }, + }; + const kv = new StateKV(sdk as never, { backend }); + const iterate = kv.iterate.bind(kv); + kv.iterate = (scope) => iterate(scope, { pageSize: 4 }); + const expected = fixture.cases.find((item) => item.name === "small5_projectA_confidence_ties")!; + expect(await register(kv)(expected.request)).toEqual({ + success: true, + insights: expected.expected[backend].map((alias) => fixture.records[alias]), + }); + }); + + it("retains the list-only fallback's original tie order", async () => { + const kv = { backend, list: async () => structuredClone(rows) }; + const expected = fixture.cases.find((item) => item.name === "small3_projectA_filters_before_selection")!; + expect(await register(kv)(expected.request)).toEqual({ + success: true, + insights: expected.expected.file.map((alias) => fixture.records[alias]), + }); + }); + + it("propagates a later page error instead of returning a partial top result", async () => { + const error = new Error("later insight page unavailable"); + const kv = { backend, async *iterate() { yield rows.slice(0, 3); throw error; } }; + await expect(register(kv)({ limit: 1 })).rejects.toBe(error); + }); + + it("keeps a small query below a retained-heap budget for large historical records", () => { + const result = spawnSync(process.execPath, [ + "--max-old-space-size=384", "--expose-gc", "--import", "tsx", + fileURLToPath(new URL("./fixtures/insight-list-memory.ts", import.meta.url)), backend, + ], { encoding: "utf8", timeout: 60_000 }); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + const metrics = JSON.parse(result.stdout.trim().split("\n").at(-1)!); + expect(metrics).toMatchObject({ backend, records: 4096, reads: 4096, modelRequests: 0 }); + expect(metrics.heapGrowthBytes).toBeLessThan(24 * 1024 * 1024); + }, 65_000); +});