diff --git a/mcp/src/graph/delta-since.integration.test.ts b/mcp/src/graph/delta-since.integration.test.ts new file mode 100644 index 000000000..27684d704 --- /dev/null +++ b/mcp/src/graph/delta-since.integration.test.ts @@ -0,0 +1,173 @@ +/** + * Integration test for the `$since` delta filter in `listQueryForLabel()` + * (exercised through the real read path: db.nodes_by_type, as used by the + * `GET /graph` endpoint). + * + * Seeds one legacy-seconds node (7-decimal string — the old Rust ingest + * format) and one new epoch-ms Integer node (what nowEpochMs() writes), then + * asserts: + * 1. a millisecond `$since` cursor older than both returns BOTH (the + * regression: the old `toFloat(...) >= $since` comparison silently + * dropped every legacy-seconds node once the frontend sent ms cursors); + * 2. a ms cursor between the two returns only the ms node (legacy seconds + * must not over-match a newer ms cursor); + * 3. mixed-format nodes sort by *normalized* ms (ORDER BY); + * 4. returned Integer timestamps are coerced to plain numbers (no + * `{low, high}` leak). + * + * Runs only against a live Neo4j at bolt://${NEO4J_HOST} (defaults + * localhost:7687 / neo4j / testtest, matching createNeo4jDriver). Skips when + * NO_DB=true or when unreachable, so `npm run test:node` stays DB-free. + * Run standalone with: + * npx tsx --test --test-timeout=60000 src/graph/delta-since.integration.test.ts + */ +import { describe, it, before, after } from "node:test"; +import assert from "node:assert/strict"; +import neo4j from "neo4j-driver"; + +import { db } from "./neo4j.js"; +import { toReturnNode } from "./utils.js"; + +const noDb = process.env.NO_DB === "true" || process.env.NO_DB === "1"; +const host = process.env.NEO4J_HOST || "localhost:7687"; +const user = process.env.NEO4J_USER || "neo4j"; +const pswd = process.env.NEO4J_PASSWORD || "testtest"; +const uri = `bolt://${host}`; + +// --- reachability probe ----------------------------------------------------- +let reachable = false; +const probeDriver = neo4j.driver(uri, neo4j.auth.basic(user, pswd), { + connectionTimeout: 3000, +}); +try { + if (!noDb) { + await probeDriver.verifyConnectivity(); + reachable = true; + console.log(`===> delta-since integration test using ${uri}`); + } +} catch { + console.log(`===> Neo4j unreachable at ${uri} — skipping integration tests`); +} +await probeDriver.close().catch(() => {}); + +const skipReason: string | false = noDb + ? "NO_DB=true" + : reachable + ? false + : `Neo4j unreachable at ${uri}`; + +// --- fixtures --------------------------------------------------------------- +const NOW_MS = Date.now(); +const RUN = `t${NOW_MS}`; +const LEGACY_KEY = `test-delta-legacy-${RUN}`; +const MS_KEY = `test-delta-ms-${RUN}`; +// legacy: epoch-seconds as a 7-decimal string (old Rust ingest format), +// stored 2h ago +const LEGACY_TS = ((NOW_MS - 2 * 3600_000) / 1000).toFixed(7); +// new: epoch-ms Integer (nowEpochMs format), stored 1h ago +const MS_TS = neo4j.int(NOW_MS - 1 * 3600_000); + +let driver: neo4j.Driver | null = null; +async function run(cypher: string, params: Record = {}) { + const session = driver!.session(); + try { + return await session.run(cypher, params); + } finally { + await session.close(); + } +} + +before(async () => { + if (!reachable) return; + driver = neo4j.driver(uri, neo4j.auth.basic(user, pswd)); + await run( + `MERGE (f:Data_Bank:Hint {node_key: $key}) + ON CREATE SET f.ref_id = $ref_id, f.name = $name, + f.date_added_to_graph = $ts`, + { key: LEGACY_KEY, ref_id: `ref-${LEGACY_KEY}`, name: "delta-legacy", ts: LEGACY_TS }, + ); + await run( + `MERGE (f:Data_Bank:Hint {node_key: $key}) + ON CREATE SET f.ref_id = $ref_id, f.name = $name, + f.date_added_to_graph = $ts`, + { key: MS_KEY, ref_id: `ref-${MS_KEY}`, name: "delta-ms", ts: MS_TS }, + ); +}); + +after(async () => { + if (!reachable) return; + await run(`MATCH (n) WHERE n.node_key IN [$lk, $mk] DETACH DELETE n`, { + lk: LEGACY_KEY, + mk: MS_KEY, + }); + await driver?.close(); +}); + +async function fetchNodes(sinceMs: number) { + return db.nodes_by_type("Hint", undefined, 50000, sinceMs); +} + +describe("delta filter: ms $since over mixed legacy-seconds / new-ms stored values", () => { + it( + "a ms cursor older than both returns the legacy-seconds AND new-ms nodes", + { skip: skipReason }, + async () => { + const nodes = await fetchNodes(NOW_MS - 3 * 3600_000); + const keys = new Set(nodes.map((n) => n.properties.node_key)); + assert.ok( + keys.has(LEGACY_KEY), + "legacy-seconds node dropped by the ms $since filter", + ); + assert.ok(keys.has(MS_KEY), "new-ms node dropped by the ms $since filter"); + }, + ); + + it( + "a ms cursor between the two returns only the new-ms node", + { skip: skipReason }, + async () => { + const nodes = await fetchNodes(NOW_MS - 1.5 * 3600_000); // between -2h and -1h + const keys = new Set(nodes.map((n) => n.properties.node_key)); + assert.ok(keys.has(MS_KEY), "ms node should match a cursor older than it"); + assert.ok( + !keys.has(LEGACY_KEY), + "legacy node must not over-match a newer ms cursor", + ); + }, + ); + + it( + "orders mixed-format nodes by normalized ms (DESC)", + { skip: skipReason }, + async () => { + const nodes = await fetchNodes(NOW_MS - 3 * 3600_000); + const msIdx = nodes.findIndex((n) => n.properties.node_key === MS_KEY); + const legacyIdx = nodes.findIndex( + (n) => n.properties.node_key === LEGACY_KEY, + ); + assert.ok(msIdx !== -1 && legacyIdx !== -1); + assert.ok( + msIdx < legacyIdx, + `expected ms node (idx ${msIdx}) before legacy node (idx ${legacyIdx})`, + ); + }, + ); + + it( + "returned Integer timestamps are coerced to plain numbers (no {low, high} leak)", + { skip: skipReason }, + async () => { + const nodes = await fetchNodes(NOW_MS - 3 * 3600_000); + const msNode = nodes.find((n) => n.properties.node_key === MS_KEY)!; + assert.ok(msNode, "ms node not found"); + const ts = msNode.properties.date_added_to_graph as unknown; + assert.equal(typeof ts, "number", `expected number, got ${typeof ts}`); + assert.equal(ts, MS_TS.toNumber()); + // The shaped API response (toReturnNode) must never carry a raw + // Integer object for this field. (Node.identity is also an Integer + // internally, but it is not part of the ReturnNode wire format.) + const json = JSON.stringify(toReturnNode(msNode)); + assert.ok(!json.includes('"low"'), `leaked Integer object: ${json}`); + }, + ); +}); diff --git a/mcp/src/graph/neo4j.ts b/mcp/src/graph/neo4j.ts index f09a4a9e6..f51a64948 100644 --- a/mcp/src/graph/neo4j.ts +++ b/mcp/src/graph/neo4j.ts @@ -29,7 +29,7 @@ import { nameFileOnly, } from "./utils.js"; import * as Q from "./queries.js"; -import { nowEpochMs, epochValueToMs } from "./time.js"; +import { nowEpochMs, toEpochMs, epochValueToMs } from "./time.js"; import { vectorizeCodeDocument, vectorizeQuery } from "../vector/index.js"; import { v4 as uuidv4 } from "uuid"; import { createByModelName } from "@microsoft/tiktokenizer"; @@ -170,8 +170,10 @@ class Db { return results .flat() .sort((a, b) => { - const at = a.properties.date_added_to_graph || 0; - const bt = b.properties.date_added_to_graph || 0; + // Normalize mixed legacy-seconds/new-ms stored values before sorting + // (backfill shim — remove once the data migration has run). + const at = toEpochMs(a.properties.date_added_to_graph) ?? 0; + const bt = toEpochMs(b.properties.date_added_to_graph) ?? 0; return bt - at; }) .slice(0, limit_total); diff --git a/mcp/src/graph/queries.test.ts b/mcp/src/graph/queries.test.ts index 56b61a2b7..cdb811eac 100644 --- a/mcp/src/graph/queries.test.ts +++ b/mcp/src/graph/queries.test.ts @@ -1,37 +1,33 @@ /** - * Unit tests for `listQueryForLabel`'s `$since` filter after the - * `toFloat(date_added_to_graph)` coercion shim removal. + * Unit tests for `listQueryForLabel`'s `$since` filter with mixed stored + * timestamp formats. * - * `date_added_to_graph` is a canonical epoch-ms Integer (see ./time.ts), so - * the property compares directly against an epoch-ms `$since`. These tests: - * 1. pin the generated query text to the bare, coercion-free predicate; - * 2. prove the clause's selection semantics on ms-magnitude fixtures — - * a recently-added node is selected, a stale one is not; - * 3. prove a seconds-magnitude `$since` must be normalized (×1000) before - * binding — which `nodes_by_type` does via `epochValueToMs`. + * Stored `date_added_to_graph` values may be legacy epoch-seconds or new + * epoch-ms Integers until the data backfill migration. The generated query + * therefore normalizes the stored property via `epochMsExpr` (Cypher mirror + * of `toEpochMs`) before comparing against an epoch-ms `$since`. + * + * `nodes_by_type` also normalizes a seconds-magnitude caller `$since` via + * `epochValueToMs` before binding. * * Runs under NO_DB=true — no Neo4j contacted. */ import { describe, it } from "node:test"; import assert from "node:assert/strict"; -import { listQueryForLabel } from "./queries.js"; -import { epochValueToMs } from "./time.js"; +import { epochMsExpr, listQueryForLabel } from "./queries.js"; +import { epochValueToMs, toEpochMs } from "./time.js"; const HOUR_MS = 3600 * 1000; -describe("listQueryForLabel $since filter (post toFloat removal)", () => { - it("compares date_added_to_graph directly — no toFloat coercion remains", () => { +describe("listQueryForLabel $since filter (mixed timestamp formats)", () => { + it("normalizes stored date_added_to_graph via epochMsExpr before comparing", () => { const q = listQueryForLabel("Function", true); - assert.doesNotMatch( - q, - /toFloat\s*\(/, - "toFloat coercion shim must be gone" - ); - assert.match(q, /f\.date_added_to_graph >= \$since/); + assert.match(q, /CASE WHEN toFloat\(f\.date_added_to_graph\) <= 1000000000000/); + assert.match(q, /f\.date_added_to_graph\) END >= \$since/); assert.match( q, - /ORDER BY coalesce\(f\.date_added_to_graph, 0\) DESC, f\.node_key/ + /ORDER BY CASE WHEN toFloat\(coalesce\(f\.date_added_to_graph, 0\)\) <= 1000000000000/, ); }); @@ -43,14 +39,14 @@ describe("listQueryForLabel $since filter (post toFloat removal)", () => { }); /** - * Mirror of the generated clause: - * `$since IS NULL OR (f.date_added_to_graph IS NOT NULL - * AND f.date_added_to_graph >= $since)` - * The text assertions above pin the template to this exact predicate, so - * the mirror cannot silently drift from what the query actually does. + * Mirror of the generated clause after both sides are normalized to ms: + * `$since IS NULL OR (storedMs IS NOT NULL AND storedMs >= sinceMs)` */ function selects(nodeDate: number | null, sinceMs: number | null): boolean { - return sinceMs === null || (nodeDate !== null && nodeDate >= sinceMs); + if (sinceMs === null) return true; + if (nodeDate === null) return false; + const storedMs = toEpochMs(nodeDate); + return storedMs !== null && storedMs >= sinceMs; } it("selects recently-added (ms-magnitude) nodes and excludes stale ones", () => { @@ -75,6 +71,27 @@ describe("listQueryForLabel $since filter (post toFloat removal)", () => { ); }); + it("a millisecond $since cursor returns both legacy-seconds and new-ms nodes", () => { + const now = Date.now(); + const sinceMs = now - 24 * HOUR_MS; + const legacySeconds = (now - 1 * HOUR_MS) / 1000; // 1h ago, stored as seconds + const newMs = now - 2 * HOUR_MS; // 2h ago, stored as ms + const tooOldSeconds = (now - 72 * HOUR_MS) / 1000; + + assert.ok( + selects(legacySeconds, sinceMs), + "legacy-seconds node within the window must match a ms cursor" + ); + assert.ok( + selects(newMs, sinceMs), + "new-ms node within the window must match the same cursor" + ); + assert.ok( + !selects(tooOldSeconds, sinceMs), + "legacy-seconds node outside the window must still be excluded" + ); + }); + it("normalizes a seconds-magnitude $since before comparing (as nodes_by_type does)", () => { const now = Date.now(); const fresh = now - 1 * HOUR_MS; @@ -98,4 +115,11 @@ describe("listQueryForLabel $since filter (post toFloat removal)", () => { assert.ok(selects(fresh, sinceMs), "fresh node still selected"); assert.ok(!selects(stale, sinceMs), "stale node excluded again"); }); + + it("epochMsExpr uses the same <= 1e12 boundary as toEpochMs", () => { + assert.equal( + epochMsExpr("f.date_added_to_graph"), + "CASE WHEN toFloat(f.date_added_to_graph) <= 1000000000000 THEN toFloat(f.date_added_to_graph) * 1000 ELSE toFloat(f.date_added_to_graph) END", + ); + }); }); diff --git a/mcp/src/graph/queries.ts b/mcp/src/graph/queries.ts index 7247b3194..5724a850d 100644 --- a/mcp/src/graph/queries.ts +++ b/mcp/src/graph/queries.ts @@ -154,7 +154,7 @@ MATCH (l:Learning) OPTIONAL MATCH (l)-[:HAS_SCOPE]->(s:Scope) WITH l, collect(s.name) AS scopes RETURN l, scopes -ORDER BY l.date_added_to_graph DESC +ORDER BY ${epochMsExpr("l.date_added_to_graph")} DESC `; export const GET_ALL_SCOPES_QUERY = ` @@ -693,19 +693,35 @@ WHERE file.name ENDS WITH 'Cargo.toml' RETURN DISTINCT file `; +/** + * Cypher mirror of `toEpochMs()` in `time.ts` — keep in sync. Normalizes a + * stored `date_added_to_graph` (legacy seconds float/string, or new epoch-ms + * Integer) to epoch milliseconds before comparing/sorting: values ≤ 1e12 are + * epoch-seconds (×1000), > 1e12 are already ms. `toFloat` handles the legacy + * 7-decimal strings, plain numbers, and Neo4j Integers alike. Load-bearing + * until the data backfill migration — without it a ms `$since` cursor would + * silently drop every legacy-seconds node. + * + * `nodes_by_type` also normalizes caller-supplied `since` values to ms via + * `epochValueToMs` before binding, so both sides of the comparison are ms. + */ +export function epochMsExpr(prop: string): string { + return `CASE WHEN toFloat(${prop}) <= 1000000000000 THEN toFloat(${prop}) * 1000 ELSE toFloat(${prop}) END`; +} + export function listQueryForLabel( label: string, withSince: boolean = false, ): string { - // `date_added_to_graph` is a canonical epoch-ms Integer (see ./time.ts), so - // it compares directly against an epoch-ms `$since` — no toFloat coercion. - // `nodes_by_type` normalizes caller-supplied `since` values to ms via - // `epochValueToMs` before binding. const sinceClause = withSince - ? `AND ($since IS NULL OR (f.date_added_to_graph IS NOT NULL AND f.date_added_to_graph >= $since))` + ? `AND ($since IS NULL OR (f.date_added_to_graph IS NOT NULL AND ${epochMsExpr( + "f.date_added_to_graph", + )} >= $since))` : ""; const orderBy = withSince - ? `ORDER BY coalesce(f.date_added_to_graph, 0) DESC, f.node_key` + ? `ORDER BY ${epochMsExpr( + "coalesce(f.date_added_to_graph, 0)", + )} DESC, f.node_key` : ""; return ` MATCH (f:${label}) diff --git a/mcp/src/graph/time.test.ts b/mcp/src/graph/time.test.ts index 64c3c637d..fd1672b49 100644 --- a/mcp/src/graph/time.test.ts +++ b/mcp/src/graph/time.test.ts @@ -3,7 +3,13 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; import neo4j from "neo4j-driver"; -import { nowEpochMs, epochValueToMs, dateAddedAgeHours } from "./time.js"; +import { + nowEpochMs, + toEpochMs, + epochValueToMs, + nodeAgeHours, + dateAddedAgeHours, +} from "./time.js"; import { ADD_NODE_QUERY, CREATE_AGENT_SESSION_STUB_QUERY, @@ -11,6 +17,8 @@ import { CREATE_MOCK_QUERY, CREATE_PROMPT_QUERY, CREATE_PULL_REQUEST_QUERY, + GET_ALL_LEARNINGS_WITH_SCOPES_QUERY, + listQueryForLabel, UPDATE_REPO_DOCS_QUERY, UPSERT_AGENT_SESSION_QUERY, UPSERT_LEARNING_QUERY, @@ -18,6 +26,7 @@ import { UPSERT_TURNS_QUERY, UPSERT_WORKFLOW_DOCUMENTATION_QUERY, } from "./queries.js"; +import { toReturnNode, toReturnNodeNoBody } from "./utils.js"; import { prepareAgeNode, prepareCommitsNode, @@ -112,6 +121,143 @@ describe("date_added_to_graph set-once semantics (query templates)", () => { }); }); +describe("toEpochMs (magnitude discriminator)", () => { + it("treats <= 1e12 as legacy epoch-seconds and converts to ms", () => { + assert.equal(toEpochMs(1700000000), 1700000000000); + // just under the threshold is still seconds + assert.equal(toEpochMs(999999999999), 999999999999000); + // exactly 10**12 is still-seconds (matches TimeFormatter.epoch_value_to_ms) + assert.equal(toEpochMs(1e12), 1e15); + }); + + it("passes > 1e12 through unchanged (already ms)", () => { + assert.equal(toEpochMs(1700000000000), 1700000000000); + assert.equal(toEpochMs(1e12 + 1), 1e12 + 1); + }); + + it("parses legacy 7-decimal seconds strings (old Rust ingest format)", () => { + assert.equal(toEpochMs("1700000000.1234567"), 1700000000123); + }); + + it("parses ms-magnitude strings", () => { + assert.equal(toEpochMs("1700000000000"), 1700000000000); + }); + + it("handles raw Neo4j Integer objects ({low, high})", () => { + const int = neo4j.int(1700000000000); + assert.equal(toEpochMs(int), 1700000000000); + }); + + it("returns null for null/undefined/empty/unparseable input", () => { + assert.equal(toEpochMs(null), null); + assert.equal(toEpochMs(undefined), null); + assert.equal(toEpochMs(""), null); + assert.equal(toEpochMs("not-a-timestamp"), null); + assert.equal(toEpochMs({} as any), null); + }); +}); + +describe("nodeAgeHours (cache-age over mixed stored formats)", () => { + // Fixed "now" in ms (~2027) so expectations are exact. + const NOW_MS = 1_800_000_000_000; + + it("ages a legacy-seconds stored value correctly", () => { + // stored 2h ago as epoch-seconds + assert.equal(nodeAgeHours((NOW_MS - 2 * 3600_000) / 1000, NOW_MS), 2); + }); + + it("ages a legacy 7-decimal seconds string correctly", () => { + const stored = ((NOW_MS - 3 * 3600_000) / 1000).toFixed(7); + assert.equal(nodeAgeHours(stored, NOW_MS), 3); + }); + + it("ages a new epoch-ms stored value correctly", () => { + assert.equal(nodeAgeHours(NOW_MS - 3600_000, NOW_MS), 1); + }); + + it("returns null for missing/unparseable values", () => { + assert.equal(nodeAgeHours(null, NOW_MS), null); + assert.equal(nodeAgeHours(undefined, NOW_MS), null); + assert.equal(nodeAgeHours("garbage", NOW_MS), null); + }); +}); + +describe("listQueryForLabel delta filter (ms cursor over mixed stored formats)", () => { + it("normalizes the stored value before the $since comparison", () => { + const q = listQueryForLabel("Hint", true); + assert.match( + q, + /CASE WHEN toFloat\(f\.date_added_to_graph\) <= 1000000000000/, + ); + assert.match(q, /toFloat\(f\.date_added_to_graph\) \* 1000 ELSE toFloat\(f\.date_added_to_graph\) END >= \$since/); + // the legacy bare-seconds comparison must be gone + assert.ok( + !q.includes("toFloat(f.date_added_to_graph) >= $since"), + "legacy comparison would silently drop all legacy-seconds nodes for a ms cursor", + ); + }); + + it("normalizes the paired ORDER BY too", () => { + const q = listQueryForLabel("Hint", true); + assert.match( + q, + /ORDER BY CASE WHEN toFloat\(coalesce\(f\.date_added_to_graph, 0\)\) <= 1000000000000/, + ); + assert.match(q, /DESC, f\.node_key/); + assert.ok(!q.includes("coalesce(toFloat(f.date_added_to_graph), 0)")); + }); + + it("omits since clauses when withSince=false", () => { + const q = listQueryForLabel("Hint", false); + assert.ok(!q.includes("$since")); + assert.ok(!q.includes("ORDER BY")); + }); + + it("normalizes the learnings ORDER BY (mixed-format sort)", () => { + assert.match( + GET_ALL_LEARNINGS_WITH_SCOPES_QUERY, + /ORDER BY CASE WHEN toFloat\(l\.date_added_to_graph\) <= 1000000000000/, + ); + }); +}); + +describe("API responses never leak raw Neo4j Integer objects", () => { + const int = neo4j.int(1700000000000); + const rawNode = { + labels: ["Data_Bank", "Hint"], + properties: { + ref_id: "r1", + name: "n", + body: "b", + date_added_to_graph: int, + }, + } as any; + + it("toReturnNode coerces {low, high} to a plain number", () => { + const ret = toReturnNode(rawNode); + assert.equal(ret.date_added_to_graph, 1700000000000); + assert.equal(typeof ret.date_added_to_graph, "number"); + assert.equal(typeof ret.properties.date_added_to_graph, "number"); + const json = JSON.stringify(ret); + assert.ok(!json.includes('"low"'), `leaked Integer object: ${json}`); + }); + + it("toReturnNodeNoBody coerces too", () => { + const ret = toReturnNodeNoBody(rawNode); + assert.equal(ret.date_added_to_graph, 1700000000000); + const json = JSON.stringify(ret); + assert.ok(!json.includes('"low"'), `leaked Integer object: ${json}`); + }); + + it("is idempotent for already-plain number values", () => { + const ret = toReturnNode({ + ...rawNode, + properties: { ...rawNode.properties, date_added_to_graph: 1700000000000 }, + }); + assert.equal(ret.date_added_to_graph, 1700000000000); + }); +}); + describe("gitsee node prep leaves timestamping to the write path", () => { const builders: Array<[string, () => { node_data: Record }]> = [ @@ -178,6 +324,12 @@ describe("dateAddedAgeHours (intelligence cache-control age math)", () => { assert.equal(dateAddedAgeHours(stamp, now), 3); }); + it("ages a legacy-seconds stored value correctly", () => { + const now = 1_800_000_000_000; + const stampSeconds = (now - 2 * HOUR_MS) / 1000; + assert.equal(dateAddedAgeHours(stampSeconds, now), 2); + }); + it("a fresh node (1h old) is within a 24h maxAgeHours window", () => { const now = Date.now(); const stamp = now - 1 * HOUR_MS; diff --git a/mcp/src/graph/time.ts b/mcp/src/graph/time.ts index f856fa840..259edef93 100644 --- a/mcp/src/graph/time.ts +++ b/mcp/src/graph/time.ts @@ -20,31 +20,76 @@ export function nowEpochMs(): Integer { * still epoch-seconds. Must match jarvis-backend's * `TimeFormatter.epoch_value_to_ms` boundary exactly (values ≤ 10**12 are * seconds) — do not introduce a divergent variant. + * + * Mirrors the Cypher expression in `queries.ts#epochMsExpr` — keep in sync. */ export const EPOCH_MS_BOUNDARY = 1_000_000_000_000; +export type EpochInput = + | number + | string + | { low: number; high?: number } + | null + | undefined; + +/** + * Magnitude discriminator: normalize any stored `date_added_to_graph` value + * to epoch **milliseconds**. + * + * The graph holds a mix of legacy formats until the data backfill migration + * runs (7-decimal strings, float seconds, integer seconds) alongside new + * epoch-ms Integers. Rule (defined ONCE here — do not reinvent per site): + * - value <= 1e12 → legacy epoch-seconds → × 1000 + * - value > 1e12 → already epoch-milliseconds → pass through + * - null / undefined / unparseable → null + * + * Sub-ms precision is truncated (`floor`) to match jarvis-backend's integer + * conversion. Also tolerates a raw Neo4j Integer (`{low, high}`) object in + * case a read path bypassed `clean_node`/`deser_node`. + */ +export function toEpochMs(value: EpochInput): number | null { + if (value === null || value === undefined) return null; + if (typeof value === "object") { + if (typeof value.low !== "number") return null; + return toEpochMs((value.high ?? 0) * 2 ** 32 + (value.low >>> 0)); + } + const n = typeof value === "number" ? value : parseFloat(value); + if (!Number.isFinite(n)) return null; + return n <= EPOCH_MS_BOUNDARY ? Math.floor(n * 1000) : Math.floor(n); +} + /** * Normalize an epoch value that may be in seconds or milliseconds to * epoch-milliseconds, mirroring jarvis-backend's - * `TimeFormatter.epoch_value_to_ms`: seconds-magnitude values (≤ 10^12) - * scale ×1000, ms-magnitude values pass through. Sub-ms precision is - * truncated (`floor`) to match the backend's integer conversion. + * `TimeFormatter.epoch_value_to_ms`. Thin wrapper over `toEpochMs` for + * numeric bind sites (e.g. `$since` in `nodes_by_type`). */ export function epochValueToMs(value: number): number { - return value <= EPOCH_MS_BOUNDARY - ? Math.floor(value * 1000) - : Math.floor(value); + return toEpochMs(value) as number; +} + +/** + * Age of a node in **hours**, given its stored `date_added_to_graph` value in + * any legacy/new format (routed through `toEpochMs`). Returns null when the + * stored value is missing or unparseable — callers decide the fallback. + */ +export function nodeAgeHours( + nodeAge: EpochInput, + nowMs: number = Date.now(), +): number | null { + const ms = toEpochMs(nodeAge); + return ms === null ? null : (nowMs - ms) / 3_600_000; } /** - * Age in hours of a node stamped with the canonical `date_added_to_graph` - * stamp (**epoch milliseconds** — the unit `nowEpochMs()` writes). Used by the - * intelligence tools' cache-control age check. Kept here, next to the stamp - * definition, so the canonical unit lives in exactly one module. + * Age in hours of a node stamped with `date_added_to_graph`. Mixed legacy + * seconds / new-ms values are normalized via `toEpochMs` first. Used by the + * intelligence tools' cache-control age check. */ export function dateAddedAgeHours( dateAddedMs: number, now: number = Date.now(), ): number { - return (now - dateAddedMs) / (3600 * 1000); + const ms = toEpochMs(dateAddedMs) ?? dateAddedMs; + return (now - ms) / (3600 * 1000); } diff --git a/mcp/src/graph/utils.ts b/mcp/src/graph/utils.ts index 03e6b7d29..7658276fe 100644 --- a/mcp/src/graph/utils.ts +++ b/mcp/src/graph/utils.ts @@ -55,6 +55,11 @@ export function rightLabel(node: Neo4jNode): NodeType { } export function toReturnNode(node: Neo4jNode): ReturnNode { + // Defensive coercion: convert any raw Neo4j Integer (`{low, high}`) + // properties (incl. date_added_to_graph) to plain numbers so they can never + // leak into JSON responses, even on paths that bypassed + // deser_node/clean_node upstream. Idempotent for already-clean nodes. + clean_node(node); const properties = node.properties; const ref_id = IS_TEST ? "test_ref_id" : properties.ref_id || ""; delete properties.ref_id; diff --git a/mcp/src/handler/index.ts b/mcp/src/handler/index.ts index 34028ea6f..f2e984152 100644 --- a/mcp/src/handler/index.ts +++ b/mcp/src/handler/index.ts @@ -2,6 +2,7 @@ import { z } from 'zod'; import { Express } from 'express'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { MCPHttpServer } from '../tools/http.js'; +import { bearerToken } from '../tools/utils.js'; import { listConcepts, learnConcept, searchLogsHandler } from './tools.js'; function createServer(): McpServer { @@ -59,11 +60,11 @@ Example queries: const httpServer = new MCPHttpServer(createServer); export function mcp_routes(app: Express) { - app.get('/mcp', async (req, res) => { + app.get('/mcp', bearerToken, async (req, res) => { await httpServer.handleGetRequest(req, res); }); - app.post('/mcp', async (req, res) => { + app.post('/mcp', bearerToken, async (req, res) => { await httpServer.handlePostRequest(req, res); }); } diff --git a/mcp/src/tools/intelligence/index.ts b/mcp/src/tools/intelligence/index.ts index 3bdbaafb9..b0b58e58e 100644 --- a/mcp/src/tools/intelligence/index.ts +++ b/mcp/src/tools/intelligence/index.ts @@ -122,6 +122,8 @@ export async function ask_prompt( existingRefIdToReplace = top.ref_id; } else if (cacheControl?.maxAgeHours) { const nodeAge = top.properties.date_added_to_graph; + // Mixed legacy-seconds / new-ms values are normalized inside + // dateAddedAgeHours → toEpochMs until the data backfill runs. if (nodeAge) { const ageInHours = dateAddedAgeHours(nodeAge); diff --git a/mcp/web/src/graph/GraphScene.tsx b/mcp/web/src/graph/GraphScene.tsx index 5c58219be..c693e7f9d 100644 --- a/mcp/web/src/graph/GraphScene.tsx +++ b/mcp/web/src/graph/GraphScene.tsx @@ -94,6 +94,10 @@ export const GraphScene = memo(() => { const destroy = useSimulation((s) => s.destroy); const hasInitialData = useRef(false); const isFetchingRef = useRef(false); + // `since` delta cursor: sent to the server as-is (whatever magnitude the + // API surfaced). The server-side delta filter normalizes stored values + // (legacy seconds vs epoch-ms) before comparing, so mixed-format graphs + // are reconciled there — no client-side conversion needed. const latestTimestampRef = useRef(null); const fetchGraph = useCallback( diff --git a/mcp/web/src/stores/useGraphData.ts b/mcp/web/src/stores/useGraphData.ts index f187c18d3..4bfe05bac 100644 --- a/mcp/web/src/stores/useGraphData.ts +++ b/mcp/web/src/stores/useGraphData.ts @@ -231,6 +231,9 @@ export const useGraphData = create((set) => ({ ...existing.properties, ...n.properties, }; + // `date_added_to_graph` arrives as a plain number (the server coerces + // Neo4j Integers; legacy seconds strings just overwrite until the + // backfill). The delta cursor built from it is normalized server-side. if (n.date_added_to_graph != null) { existing.date_added_to_graph = n.date_added_to_graph; } diff --git a/mcp/web/tsconfig.tsbuildinfo b/mcp/web/tsconfig.tsbuildinfo new file mode 100644 index 000000000..e7e0842a9 --- /dev/null +++ b/mcp/web/tsconfig.tsbuildinfo @@ -0,0 +1 @@ +{"root":["./src/App.tsx","./src/main.tsx","./src/types.ts","./src/vite-env.d.ts","./src/components/DocViewer.tsx","./src/components/EnrichButton.tsx","./src/components/ImportanceLens.tsx","./src/components/IngestionStatus.tsx","./src/components/LayerTogglePanel.tsx","./src/components/MarkdownRenderer.tsx","./src/components/Onboarding.tsx","./src/components/ProvenanceTree.tsx","./src/components/Sidebar.tsx","./src/components/SyncButton.tsx","./src/components/chat/Chat.tsx","./src/components/chat/ChatInput.tsx","./src/components/chat/ChatMessage.tsx","./src/components/chat/Settings.tsx","./src/components/chat/ToolCallFlow.tsx","./src/components/ui/badge.tsx","./src/components/ui/button.tsx","./src/components/ui/collapsible.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/graph/GraphScene.tsx","./src/graph/config.ts","./src/graph/types.ts","./src/graph/components/Edges.tsx","./src/graph/components/Graph.tsx","./src/graph/components/LayerHoverHighlight.tsx","./src/graph/components/LayerLabels.tsx","./src/graph/components/NodeDetailsPanel.tsx","./src/graph/components/NodePoints.tsx","./src/hooks/useAgentChat.ts","./src/hooks/useApi.ts","./src/hooks/useSSE.ts","./src/lib/api.ts","./src/lib/errors.ts","./src/lib/utils.ts","./src/stores/useChat.ts","./src/stores/useGraphData.ts","./src/stores/useIngestion.ts","./src/stores/useLayerVisibility.ts","./src/stores/useServerConfig.ts","./src/stores/useSettings.ts","./src/stores/useSimulation.ts"],"errors":true,"version":"5.9.3"} \ No newline at end of file