From b3685d6a0c9058aa718eaa0096bdb428fa2eaf91 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 12:22:06 +0200 Subject: [PATCH 1/9] fix(sync): reject non-IRI durable-meta subjects at unverified peer-ingest (#1921) Durable-sync _meta ingest funnels every peer-supplied metadata quad through selectVerifiedDurableSyncQuads, but neither meta selector (selectAdmittedMetadataIndexes, selectSystemOverrideMetadataIndexes) checked that the metadata SUBJECT is an IRI. A peer could inject a blank-node _meta subject -- including one bearing an integrity predicate such as dkg:merkleRoot -- and have it persisted into a durable _meta graph, where it has no trustworthy, stable identity. Add an isIriMetaSubject guard at the top of BOTH selector loops (before the merkle/marker admission branch), dropping non-IRI (blank-node / literal) subjects and counting them. REJECT, never skolemize: a peer-injected blank node has no authenticatable identity. The guard operates on incoming peer quads only -- no store read/DROP/sweep -- so there is zero data-loss risk to already-persisted data. A single aggregated warn is emitted per selection at all three guarded call sites, and the drop count is surfaced as droppedNonIriSubjects on DurableIntegritySelection (mirroring droppedSyncControlTriples). Also repoint the DurableMetaPageFrameError oversized-subject references from #1921 to #1923 (SyncPagePolicy): #1921 is IRI-only; a valid-IRI subject bearing a giant literal is an orthogonal size vector. The #1916 responder subject-atomic docstring is updated to note ingest now enforces IRI-only for NEW peer ingest, while the store-paged blank-node accommodation stays as defense-in-depth for pre-fix persisted data. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync/durable-integrity.ts | 63 ++++++++++- .../agent/src/sync/responder/graph-plan.ts | 20 ++-- .../sync-control-metadata-admission.test.ts | 102 ++++++++++++++++++ 3 files changed, 174 insertions(+), 11 deletions(-) diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index ab6ee9f2f7..831788260b 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -104,6 +104,8 @@ export interface DurableIntegritySelection { rejected: number; /** Unauthenticated cursor/routing rows deliberately consumed but not persisted. */ droppedSyncControlTriples: number; + /** Non-IRI (blank-node/literal) `_meta` subject rows dropped at peer ingest (#1921). */ + droppedNonIriSubjects: number; /** Verified V2 assets whose exact public assertion graph is intentionally empty. */ verifiedZeroPublicAssets: number; /** Exact assertion graphs whose V2 descriptors and fetched payload verified. */ @@ -365,6 +367,7 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 1, droppedSyncControlTriples: 0, + droppedNonIriSubjects: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -375,6 +378,7 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 0, droppedSyncControlTriples: 0, + droppedNonIriSubjects: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -393,6 +397,7 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 1, droppedSyncControlTriples: 0, + droppedNonIriSubjects: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -406,11 +411,13 @@ export function selectVerifiedDurableSyncQuads( new Set(), ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjects); return { dataIndexes: allIndexes(dataQuads), metaIndexes: selectedMetadata.indexes, rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, + droppedNonIriSubjects: selectedMetadata.droppedNonIriSubjects, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -1019,11 +1026,13 @@ function selectVerifiedQuads( outcome.kaToKc, ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjects); return { dataIndexes: allIndexes(dataQuads), metaIndexes: selectedMetadata.indexes, rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, + droppedNonIriSubjects: selectedMetadata.droppedNonIriSubjects, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1035,6 +1044,7 @@ function selectVerifiedQuads( metaIndexes: [], rejected, droppedSyncControlTriples: 0, + droppedNonIriSubjects: 0, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, // Keep this list aligned with the selected data + metadata indexes. // A fatal batch deliberately selects neither. Returning the names of @@ -1054,12 +1064,14 @@ function selectVerifiedQuads( outcome.authenticatedMetadataUals, ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjects); return { dataIndexes, metaIndexes: selectedMetadata.indexes, rejected, droppedSyncControlTriples: selectedMetadata.droppedControls, + droppedNonIriSubjects: selectedMetadata.droppedNonIriSubjects, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1072,11 +1084,21 @@ function selectAdmittedMetadataIndexes( admittedMetadataUals: ReadonlySet, kaToKc: ReadonlyMap, authenticatedMetadataUals: ReadonlySet, -): { indexes: number[]; droppedControls: number } { +): { indexes: number[]; droppedControls: number; droppedNonIriSubjects: number } { const indexes: number[] = []; let droppedControls = 0; + let droppedNonIriSubjects = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; + // #1921 — a durable `_meta` subject must be a conforming IRI. Drop a + // blank-node/literal subject here, BEFORE the merkle/marker admission + // branch below: `indexIntegrityMetadata` adds ANY merkleRoot-bearing + // subject to `merkleSubjects` with no IRI check, so a forged-integrity + // blank node would otherwise reach that branch. + if (!isIriMetaSubject(quad.subject)) { + droppedNonIriSubjects += 1; + continue; + } if ( metadata.merkleSubjects.has(quad.subject) || metadata.markerSubjects.has(quad.subject) @@ -1130,7 +1152,7 @@ function selectAdmittedMetadataIndexes( } indexes.push(index); } - return { indexes, droppedControls }; + return { indexes, droppedControls, droppedNonIriSubjects }; } function selectSystemOverrideMetadataIndexes( @@ -1138,11 +1160,20 @@ function selectSystemOverrideMetadataIndexes( metadata: IntegrityMetadataIndex, authenticatedMetadataUals: ReadonlySet, kaToKc: ReadonlyMap, -): { indexes: number[]; droppedControls: number } { +): { indexes: number[]; droppedControls: number; droppedNonIriSubjects: number } { const indexes: number[] = []; let droppedControls = 0; + let droppedNonIriSubjects = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; + // #1921 — reject a non-IRI durable `_meta` subject at ingest. This terminal + // system-CG selector admits every non-control row, so without this guard a + // blank-node subject bearing ANY descriptive or integrity predicate + // (e.g. `dkg:merkleRoot`) would be persisted and later served back. + if (!isIriMetaSubject(quad.subject)) { + droppedNonIriSubjects += 1; + continue; + } if ( DURABLE_SYNC_CONTROL_PREDICATE_SET.has(quad.predicate) && !isGraphSealDescriptiveVersion(quad, metadata) @@ -1158,7 +1189,7 @@ function selectSystemOverrideMetadataIndexes( } indexes.push(index); } - return { indexes, droppedControls }; + return { indexes, droppedControls, droppedNonIriSubjects }; } /** @@ -1247,6 +1278,17 @@ function logDroppedSyncControls( }); } +function logDroppedNonIriMetaSubjects( + logs: DurableIntegrityLogEntry[], + dropped: number, +): void { + if (dropped === 0) return; + logs.push({ + level: 'warn', + message: `Dropped ${dropped} non-IRI durable _meta subject triple(s) from peer ingest (#1921)`, + }); +} + function isDetachedLegacyProjectionGraph(graph: string): boolean { if (!graph.startsWith('did:dkg:context-graph:')) return false; return graph.endsWith('/_catalog') @@ -1394,6 +1436,19 @@ function stripLiteral(raw: string): string { return match ? match[1]! : raw; } +/** + * A durable `_meta` subject must be an IRI. Conforming writers only emit IRI + * subjects (metadata generators build deterministic UALs; the publisher rejects + * blank nodes; SWM writers skolemize before storage). A blank-node (`_:…`) or + * literal (`"…"`) subject is only reachable via unverified peer-ingest and has + * no trustworthy, stable identity, so it is dropped at ingest (#1921) rather + * than persisted. Mirrors the responder's `isIriTerm` (graph-plan.ts) so ingest + * and read agree on the contract. + */ +function isIriMetaSubject(term: string): boolean { + return term.length > 0 && !term.startsWith('_:') && !term.startsWith('"'); +} + function findLegacyRootOwner(subject: string, roots: ReadonlySet): string | undefined { for (const root of roots) { if (subject === root || subject.startsWith(`${root}${SKOLEM_SUFFIX}`)) return root; diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 03673ecc26..a6ad392d99 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -613,13 +613,16 @@ type MetaOversizedSubjectPolicy = 'byte-fit' | 'fail-loud'; * partial-metadata loss + a #1788 seal split). Failing loud is strictly better * than silent loss here: a byte-budget-negotiating requester never hits this (it * uses empty=EOF pagination), and the oversized `_meta` subject itself is only - * reachable via unverified peer-ingest, fixed at the root by #1921. + * reachable via unverified peer-ingest. Closing that SIZE vector at the root is + * tracked by #1923 (SyncPagePolicy) — it is orthogonal to #1921's IRI-only + * ingest guard, since a peer can inject a valid-IRI subject bearing a giant + * literal. * * Contract: this is a HARD, NON-RETRYABLE failure. The sync responder surfaces it * as `outcome:'error'` and re-throws it (it is deliberately NOT wrapped in a * retryable error, and NEVER converted to an empty — EOF — body), because a retry * cannot make an oversized subject servable to a legacy requester; the only - * resolutions are a requester upgrade or the #1921 ingest fix. The budget it is + * resolutions are a requester upgrade or the #1923 ingest fix. The budget it is * checked against is `SYNC_BYTE_BUDGET_RESPONSE_BYTES` (the router read cap minus * the frame headroom) — the largest response body guaranteed to fit one transport * frame — so a page within it is always sendable and only a genuinely oversized @@ -636,7 +639,7 @@ export class DurableMetaPageFrameError extends Error { + `non-negotiated (legacy) requester: a subject-atomic page of ${params.bytes} bytes ` + `exceeds the ${params.limit}-byte frame budget. A byte-budget-negotiating requester ` + `(testnet-canary+) paginates this via empty=EOF; upgrade the requester or reject the ` - + `oversized _meta subject at ingest (#1921).`, + + `oversized _meta subject at ingest (#1923).`, ); this.name = 'DurableMetaPageFrameError'; this.contextGraphId = params.contextGraphId; @@ -3795,10 +3798,13 @@ async function readDurableMetaRowsPage( * but within a single self-contained window its label — and therefore the * `(g, s)` boundary comparison — is consistent. Conforming writers only emit * IRI `_meta` subjects (metadata generators build deterministic IRIs; the - * publisher rejects blank nodes), but a blank-node subject is reachable via the - * unverified system-CG peer-ingest path, so subject atomicity must cover it too; - * enforcing the IRI invariant at ingest (defense-in-depth) is tracked - * separately. + * publisher rejects blank nodes). NEW unverified peer ingest now enforces this + * IRI invariant at the durable-meta selection (#1921), so it no longer admits a + * blank-node subject — but that guard operates on incoming quads only and does + * NOT sweep already-persisted data, so a blank-node `_meta` subject written by a + * pre-fix peer may still reside in the store. Subject atomicity must therefore + * still cover it, and this per-query relabel accommodation stays as + * defense-in-depth. * * `_meta` subjects are small (a seal is 14 quads ≈ a few KB), so this converges * in about one read. The returned page is BOTH subject-atomic AND ≤ diff --git a/packages/agent/test/sync-control-metadata-admission.test.ts b/packages/agent/test/sync-control-metadata-admission.test.ts index 76d5ea18c7..c70b55d318 100644 --- a/packages/agent/test/sync-control-metadata-admission.test.ts +++ b/packages/agent/test/sync-control-metadata-admission.test.ts @@ -259,4 +259,106 @@ describe('durable sync control metadata admission', () => { expect(selectedMeta(data, meta)).toEqual(meta); }); + + // #1921 — a durable `_meta` subject must be a conforming IRI. A peer can only + // reach the unverified system-CG ingest path with a blank-node (or literal) + // subject; it has no trustworthy identity, so it is dropped (never persisted, + // never skolemized) at the selection chokepoint. The guard sits at the top of + // BOTH selector loops, so it fires on every guarded call site. + const IRI_SUBJECT = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/50'; + + it('drops a forged-integrity blank-node _meta subject on the system-override path (#1921)', () => { + // The real attack vector: a blank node bearing an INTEGRITY predicate. + // `indexIntegrityMetadata` adds ANY merkleRoot subject to `merkleSubjects` + // with no IRI check, so this reaches the merkle/marker admission machinery; + // legacy verification rejects it (no rootEntity), routing to the system + // override where every non-control row is otherwise admitted. Mirrors the + // IRI-subject case "does not let the system-graph override authenticate + // malformed legacy controls" (which KEEPS the merkleRoot) — the only + // difference is the blank-node subject, which #1921 now drops. + const injected = '_:injected'; + const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"legit"'); + const meta = [ + quad(injected, `${DKG}merkleRoot`, `"${'00'.repeat(32)}"`), + quad(injected, `${DKG}status`, '"forged"'), + quad(injected, `${DKG}batchId`, integer(999n)), + keptRow, + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); + expect(selection.droppedNonIriSubjects).toBe(3); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('drops a blank-node _meta subject before the verified merkle/marker admission branch (#1921)', () => { + // A blank node bearing a FULLY VERIFIABLE legacy envelope (merkleRoot + + // rootEntity over matching data). Verification keys on the rootEntity, not + // the KC UAL, so the blank-node subject verifies and would otherwise reach + // `selectAdmittedMetadataIndexes`' merkle/marker branch and be persisted + // (compare "preserves batch controls bound to a verified legacy envelope"). + // The loop-top guard drops it first. + const root = 'urn:legacy:root'; + const injected = '_:injected'; + const data = [quad(root, 'urn:example:value', '"legacy"', CONTEXT_GRAPH)]; + const merkleRoot = `"${toHex(computeFlatKCRootV10(data, []))}"`; + const meta = [ + quad(injected, `${DKG}merkleRoot`, merkleRoot), + quad(injected, `${DKG}rootEntity`, root), + quad(injected, `${DKG}batchId`, integer(41n)), + ]; + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + expect(selection.metaIndexes).toEqual([]); + expect(selection.droppedNonIriSubjects).toBe(3); + expect(selection.rejected).toBe(0); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('drops a non-IRI _meta subject on a descriptor-less system page (#1921)', () => { + // The :401 no-merkle/no-marker branch: `selectAdmittedMetadataIndexes` runs + // with empty admission sets, so every row falls through the descriptive + // path. Exercises the third guarded call site's warn wiring. + const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"keep"'); + const meta = [ + keptRow, + quad('_:injected', `${DKG}status`, '"drop"'), + quad('_:injected', `${DKG}label`, '"drop-too"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); + expect(selection.droppedNonIriSubjects).toBe(2); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('keeps a conforming IRI _meta subject untouched (no #1921 drop)', () => { + const scope = createGraphKnowledgeAssetScope(UAL, '1'); + const assertionGraph = knowledgeAssetLayerGraphUri( + CONTEXT_GRAPH_ID, + MemoryLayer.VerifiableMemory, + scope, + ); + const data = [quad('urn:verified:entity', 'urn:example:value', '"verified"', assertionGraph)]; + const meta = generateGraphKnowledgeAssetMetadata({ + ual: UAL, + contextGraphId: CONTEXT_GRAPH_ID, + merkleRoot: computeFlatKCRootV10(data, []), + publisherPeerId: 'verified-publisher', + accessPolicy: 'public', + timestamp: new Date(0), + assertionVersion: '1', + publicTripleCount: data.length, + assertionGraph, + }, { status: 'tentative' }); + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + expect(selection.droppedNonIriSubjects).toBe(0); + expect(selection.logs.every((entry) => !/non-IRI/.test(entry.message))).toBe(true); + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual(meta); + }); }); From 664303cdf415157db60c98656f0397b8ffa1930e Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 14:08:50 +0200 Subject: [PATCH 2/9] docs(sync): clarify why the #1921 IRI guard precedes merkle-admission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment-only. Expand the loop-top guard comment in selectAdmittedMetadataIndexes to spell out why the IRI-only check must run BEFORE (not inside) the merkle/marker admission branch: indexIntegrityMetadata adds any merkleRoot-bearing subject to merkleSubjects without validating the term, and a persisted blank-node subject has no stable identity — the responder must later serve it back, where #1916's store-paged subject-atomic lane only stays sound via Oxigraph's per-query relabel. No logic change. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync/durable-integrity.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index 831788260b..e3341f53bb 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -1093,8 +1093,14 @@ function selectAdmittedMetadataIndexes( // #1921 — a durable `_meta` subject must be a conforming IRI. Drop a // blank-node/literal subject here, BEFORE the merkle/marker admission // branch below: `indexIntegrityMetadata` adds ANY merkleRoot-bearing - // subject to `merkleSubjects` with no IRI check, so a forged-integrity - // blank node would otherwise reach that branch. + // subject to `merkleSubjects` without validating the subject term, so a + // forged-integrity blank node would otherwise fall into that branch and be + // persisted. The ordering is load-bearing — the check must run before the + // branch, not inside it: a blank node has no stable identity, and once + // persisted the responder must serve it back, where #1916's store-paged + // subject-atomic lane only stays sound for it via Oxigraph's per-query + // relabel. Rejecting at the loop top preserves the store invariant that + // every durable `_meta` subject is a conforming IRI. if (!isIriMetaSubject(quad.subject)) { droppedNonIriSubjects += 1; continue; From fd6990006c1fa8f305526f9a8f4d442fa5a812d8 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 15:12:25 +0200 Subject: [PATCH 3/9] fix(sync): advance meta cursor when a page is fully dropped as non-IRI (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the otReviewAgent review of PR #1936. Bug (liveness): the durable requester advances the meta checkpoint for a metadata-only page that was ENTIRELY discarded only when the discarded rows are counted as deliberately consumed — durable-sync.ts gated that on droppedSyncControlTriples === totalFetchedMetaQuads. A page consisting solely of non-IRI `_meta` rows dropped by the #1921 ingest guard produced droppedSyncControlTriples=0, so updateMetaCheckpoint stayed false and the meta cursor pinned: the same page is re-fetched and re-dropped every round. This is also a regression vs pre-#1921 (those rows used to be persisted, so verifiedMeta>0 advanced the cursor). Reachable when a responder holds pre-fix persisted non-IRI rows or a malicious responder injects them on the wire. Fix: thread droppedNonIriSubjectTriples through the worker boundary (DurableBatchProcessResult -> wire result -> requester) and generalize the requester check to deliberatelyDroppedMeta = droppedSyncControlTriples + droppedNonIriSubjectTriples, compared to totalFetchedMetaQuads. This also fixes a MIXED all-discarded page (some controls + some non-IRI) that pins today because neither count alone equals the fetched total. Also folds in the review's other points: - Rename droppedNonIriSubjects -> droppedNonIriSubjectTriples (it counts dropped rows, mirroring droppedSyncControlTriples). - Add a literal-subject ingest-drop test (guard rejects blank-node AND literal). - Add requester-path regression tests (sync-requester-progress.test.ts) proving the meta cursor advances on an all-non-IRI page and a mixed all-discarded page; verified to fail without the propagate fix. Deferred: unifying the ingest/responder IRI term classifier -> #1940. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync-verify-worker-impl.ts | 5 + packages/agent/src/sync-verify-worker.ts | 8 ++ packages/agent/src/sync/durable-integrity.ts | 38 ++++---- .../agent/src/sync/requester/durable-sync.ts | 14 ++- .../sync-control-metadata-admission.test.ts | 26 +++++- .../test/sync-requester-progress.test.ts | 93 +++++++++++++++++++ 6 files changed, 159 insertions(+), 25 deletions(-) diff --git a/packages/agent/src/sync-verify-worker-impl.ts b/packages/agent/src/sync-verify-worker-impl.ts index 6f42cdf6fc..f12ec2fa7a 100644 --- a/packages/agent/src/sync-verify-worker-impl.ts +++ b/packages/agent/src/sync-verify-worker-impl.ts @@ -307,6 +307,7 @@ function processDurableBatch( verifiedMetaIndexes: [], verifiedGraphScopedDataGraphs: [], droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, verifiedPrivateOnlyResponses: 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -330,6 +331,7 @@ function processDurableBatch( verifiedMetaIndexes: [], verifiedGraphScopedDataGraphs: [], droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, verifiedPrivateOnlyResponses: 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -384,6 +386,7 @@ function processDurableBatch( verifiedMetaIndexes: verifiedSelection.metaIndexes, verifiedGraphScopedDataGraphs: verifiedSelection.verifiedGraphScopedDataGraphs, droppedSyncControlTriples: verifiedSelection.droppedSyncControlTriples, + droppedNonIriSubjectTriples: verifiedSelection.droppedNonIriSubjectTriples, verifiedPrivateOnlyResponses: verifiedFullyPrivateResponse ? 1 : 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -423,6 +426,7 @@ export function processDurableBatchForWire( verifiedMetaIndexes, verifiedGraphScopedDataGraphs, droppedSyncControlTriples, + droppedNonIriSubjectTriples, verifiedPrivateOnlyResponses, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -439,6 +443,7 @@ export function processDurableBatchForWire( verifiedMetaIndexes, verifiedGraphScopedDataGraphs, droppedSyncControlTriples, + droppedNonIriSubjectTriples, verifiedPrivateOnlyResponses, totalFetchedDataQuads, totalFetchedMetaQuads, diff --git a/packages/agent/src/sync-verify-worker.ts b/packages/agent/src/sync-verify-worker.ts index 83107e99bf..07cf172be9 100644 --- a/packages/agent/src/sync-verify-worker.ts +++ b/packages/agent/src/sync-verify-worker.ts @@ -48,6 +48,14 @@ export interface DurableBatchProcessResult { verifiedGraphScopedDataGraphs: string[]; /** Metadata controls deliberately consumed after failing authentication. */ droppedSyncControlTriples: number; + /** + * Non-IRI (blank-node/literal) `_meta` subject rows deliberately dropped at + * ingest (#1921). Surfaced beside {@link droppedSyncControlTriples} so the + * requester counts them as consumed metadata for meta-checkpoint advancement + * (a metadata-only page discarded entirely by the guard must still advance + * the cursor, not pin durable sync on the same page). + */ + droppedNonIriSubjectTriples: number; /** Clean batches containing verified V2 assets with no public assertion triples. */ verifiedPrivateOnlyResponses: number; totalFetchedDataQuads: number; diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index e3341f53bb..a889e31e46 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -105,7 +105,7 @@ export interface DurableIntegritySelection { /** Unauthenticated cursor/routing rows deliberately consumed but not persisted. */ droppedSyncControlTriples: number; /** Non-IRI (blank-node/literal) `_meta` subject rows dropped at peer ingest (#1921). */ - droppedNonIriSubjects: number; + droppedNonIriSubjectTriples: number; /** Verified V2 assets whose exact public assertion graph is intentionally empty. */ verifiedZeroPublicAssets: number; /** Exact assertion graphs whose V2 descriptors and fetched payload verified. */ @@ -367,7 +367,7 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 1, droppedSyncControlTriples: 0, - droppedNonIriSubjects: 0, + droppedNonIriSubjectTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -378,7 +378,7 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 0, droppedSyncControlTriples: 0, - droppedNonIriSubjects: 0, + droppedNonIriSubjectTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -397,7 +397,7 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 1, droppedSyncControlTriples: 0, - droppedNonIriSubjects: 0, + droppedNonIriSubjectTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -411,13 +411,13 @@ export function selectVerifiedDurableSyncQuads( new Set(), ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); - logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjects); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjectTriples); return { dataIndexes: allIndexes(dataQuads), metaIndexes: selectedMetadata.indexes, rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, - droppedNonIriSubjects: selectedMetadata.droppedNonIriSubjects, + droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -1026,13 +1026,13 @@ function selectVerifiedQuads( outcome.kaToKc, ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); - logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjects); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjectTriples); return { dataIndexes: allIndexes(dataQuads), metaIndexes: selectedMetadata.indexes, rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, - droppedNonIriSubjects: selectedMetadata.droppedNonIriSubjects, + droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1044,7 +1044,7 @@ function selectVerifiedQuads( metaIndexes: [], rejected, droppedSyncControlTriples: 0, - droppedNonIriSubjects: 0, + droppedNonIriSubjectTriples: 0, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, // Keep this list aligned with the selected data + metadata indexes. // A fatal batch deliberately selects neither. Returning the names of @@ -1064,14 +1064,14 @@ function selectVerifiedQuads( outcome.authenticatedMetadataUals, ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); - logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjects); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjectTriples); return { dataIndexes, metaIndexes: selectedMetadata.indexes, rejected, droppedSyncControlTriples: selectedMetadata.droppedControls, - droppedNonIriSubjects: selectedMetadata.droppedNonIriSubjects, + droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1084,10 +1084,10 @@ function selectAdmittedMetadataIndexes( admittedMetadataUals: ReadonlySet, kaToKc: ReadonlyMap, authenticatedMetadataUals: ReadonlySet, -): { indexes: number[]; droppedControls: number; droppedNonIriSubjects: number } { +): { indexes: number[]; droppedControls: number; droppedNonIriSubjectTriples: number } { const indexes: number[] = []; let droppedControls = 0; - let droppedNonIriSubjects = 0; + let droppedNonIriSubjectTriples = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; // #1921 — a durable `_meta` subject must be a conforming IRI. Drop a @@ -1102,7 +1102,7 @@ function selectAdmittedMetadataIndexes( // relabel. Rejecting at the loop top preserves the store invariant that // every durable `_meta` subject is a conforming IRI. if (!isIriMetaSubject(quad.subject)) { - droppedNonIriSubjects += 1; + droppedNonIriSubjectTriples += 1; continue; } if ( @@ -1158,7 +1158,7 @@ function selectAdmittedMetadataIndexes( } indexes.push(index); } - return { indexes, droppedControls, droppedNonIriSubjects }; + return { indexes, droppedControls, droppedNonIriSubjectTriples }; } function selectSystemOverrideMetadataIndexes( @@ -1166,10 +1166,10 @@ function selectSystemOverrideMetadataIndexes( metadata: IntegrityMetadataIndex, authenticatedMetadataUals: ReadonlySet, kaToKc: ReadonlyMap, -): { indexes: number[]; droppedControls: number; droppedNonIriSubjects: number } { +): { indexes: number[]; droppedControls: number; droppedNonIriSubjectTriples: number } { const indexes: number[] = []; let droppedControls = 0; - let droppedNonIriSubjects = 0; + let droppedNonIriSubjectTriples = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; // #1921 — reject a non-IRI durable `_meta` subject at ingest. This terminal @@ -1177,7 +1177,7 @@ function selectSystemOverrideMetadataIndexes( // blank-node subject bearing ANY descriptive or integrity predicate // (e.g. `dkg:merkleRoot`) would be persisted and later served back. if (!isIriMetaSubject(quad.subject)) { - droppedNonIriSubjects += 1; + droppedNonIriSubjectTriples += 1; continue; } if ( @@ -1195,7 +1195,7 @@ function selectSystemOverrideMetadataIndexes( } indexes.push(index); } - return { indexes, droppedControls, droppedNonIriSubjects }; + return { indexes, droppedControls, droppedNonIriSubjectTriples }; } /** diff --git a/packages/agent/src/sync/requester/durable-sync.ts b/packages/agent/src/sync/requester/durable-sync.ts index a710dcfdca..7a229234b7 100644 --- a/packages/agent/src/sync/requester/durable-sync.ts +++ b/packages/agent/src/sync/requester/durable-sync.ts @@ -118,6 +118,7 @@ interface DurableSyncContext { verifiedMeta: Quad[]; verifiedGraphScopedDataGraphs?: string[]; droppedSyncControlTriples?: number; + droppedNonIriSubjectTriples?: number; totalFetchedDataQuads: number; totalFetchedMetaQuads: number; rejectedKcs: number; @@ -473,11 +474,20 @@ export async function runDurableSync( const metadataOnlyResponse = processed.metaOnlyResponses > 0; const droppedSyncControlTriples = processed.droppedSyncControlTriples ?? 0; + // Rows the verifier deliberately consumed but did not persist: unverified + // sync controls (existing) plus non-IRI `_meta` subjects dropped at ingest + // (#1921). A metadata-only page discarded ENTIRELY by these guards carries + // no verifiedMeta, so — like the all-controls case — the meta cursor only + // advances if we still count the page as consumed. Summing both kinds also + // covers a MIXED all-discarded page (some controls + some non-IRI), which + // otherwise pins because neither count alone equals the fetched total. + const deliberatelyDroppedMeta = + droppedSyncControlTriples + (processed.droppedNonIriSubjectTriples ?? 0); const discardedOnlyMetadataResponse = metadataOnlyResponse && processed.verifiedData.length === 0 && processed.verifiedMeta.length === 0 - && droppedSyncControlTriples > 0 - && droppedSyncControlTriples === processed.totalFetchedMetaQuads; + && deliberatelyDroppedMeta > 0 + && deliberatelyDroppedMeta === processed.totalFetchedMetaQuads; const updateMetaCheckpoint = batchVerifiedCleanly && processed.dataRejectedMissingMeta === 0 && ( diff --git a/packages/agent/test/sync-control-metadata-admission.test.ts b/packages/agent/test/sync-control-metadata-admission.test.ts index c70b55d318..e67c8a89be 100644 --- a/packages/agent/test/sync-control-metadata-admission.test.ts +++ b/packages/agent/test/sync-control-metadata-admission.test.ts @@ -288,7 +288,7 @@ describe('durable sync control metadata admission', () => { const selection = selectVerifiedDurableSyncQuads([], meta, true); expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); - expect(selection.droppedNonIriSubjects).toBe(3); + expect(selection.droppedNonIriSubjectTriples).toBe(3); expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); @@ -312,7 +312,7 @@ describe('durable sync control metadata admission', () => { const selection = selectVerifiedDurableSyncQuads(data, meta, false); expect(selection.metaIndexes).toEqual([]); - expect(selection.droppedNonIriSubjects).toBe(3); + expect(selection.droppedNonIriSubjectTriples).toBe(3); expect(selection.rejected).toBe(0); expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); @@ -331,7 +331,7 @@ describe('durable sync control metadata admission', () => { const selection = selectVerifiedDurableSyncQuads([], meta, true); expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); - expect(selection.droppedNonIriSubjects).toBe(2); + expect(selection.droppedNonIriSubjectTriples).toBe(2); expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); @@ -357,8 +357,26 @@ describe('durable sync control metadata admission', () => { const selection = selectVerifiedDurableSyncQuads(data, meta, false); - expect(selection.droppedNonIriSubjects).toBe(0); + expect(selection.droppedNonIriSubjectTriples).toBe(0); expect(selection.logs.every((entry) => !/non-IRI/.test(entry.message))).toBe(true); expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual(meta); }); + + it('drops a literal `_meta` subject at ingest (#1921)', () => { + // The guard rejects BOTH blank-node and literal subjects as non-IRI. A + // literal subject term (begins with `"`) is malformed for a `_meta` subject + // and must be dropped, counted, and warned exactly like a blank node — this + // proves the second half of the advertised ingest contract. + const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"keep"'); + const meta = [ + keptRow, + quad('"literal-subject"', `${DKG}status`, '"drop"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); + expect(selection.droppedNonIriSubjectTriples).toBe(1); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); }); diff --git a/packages/agent/test/sync-requester-progress.test.ts b/packages/agent/test/sync-requester-progress.test.ts index 5916c69df7..a88e6f19d1 100644 --- a/packages/agent/test/sync-requester-progress.test.ts +++ b/packages/agent/test/sync-requester-progress.test.ts @@ -571,6 +571,99 @@ describe('sync requester progress accounting', () => { expect(setCheckpoint.calls).toEqual([]); }); + it('advances the meta cursor when the whole page is non-IRI subjects dropped at ingest (#1921)', async () => { + // A metadata-only page consisting solely of non-IRI `_meta` rows is fully + // discarded by the #1921 guard: verifiedMeta=[] and droppedSyncControlTriples=0, + // but droppedNonIriSubjectTriples === totalFetchedMetaQuads. The meta cursor + // must still advance (the rows were deliberately consumed) or durable sync + // pins on the same poisoned page. Fails before the propagate fix. + const setCheckpoint = recorder((_key: string, _offset: number) => {}); + const deleteCheckpoint = recorder((_key: string) => {}); + const storeInsert = recorder(async (_quads: Quad[]) => {}); + const fetchSyncPages = recorder(async ( + _ctx: OperationContext, + _peer: string, + contextGraphId: string, + _includeSharedMemory: boolean, + phase: 'data' | 'meta', + ) => phase === 'meta' + ? pageResult(contextGraphId, phase, { nextOffset: 3, completed: false }) + : pageResult(contextGraphId, phase)); + + const summary = await runDurableSync({ + ctx, + remotePeerId: 'peer-a', + contextGraphIds: ['discarded-non-iri'], + createContextGraphSyncDeadline: () => Date.now() + 60_000, + fetchSyncPages, + processDurableBatchInWorker: async () => ({ + ...durableProcessResult(), + emptyResponses: 0, + metaOnlyResponses: 1, + droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 3, + totalFetchedMetaQuads: 3, + }), + storeInsert, + deleteCheckpoint, + setCheckpoint, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(summary.metaOnlyResponses).toBe(1); + expect(summary.checkpointAdvances).toBe(0); + expect(storeInsert.calls).toEqual([]); + expect(deleteCheckpoint.calls).toEqual([]); + expect(setCheckpoint.calls).toEqual([['discarded-non-iri:meta', 3]]); + }); + + it('advances the meta cursor when a mixed page is fully discarded by controls + non-IRI drops (#1921)', async () => { + // A mixed all-discarded page (some unverified controls + some non-IRI rows) + // also pins today because neither count alone equals the fetched total. The + // summed deliberate-drop check advances the cursor. Fails before the fix. + const setCheckpoint = recorder((_key: string, _offset: number) => {}); + const deleteCheckpoint = recorder((_key: string) => {}); + const storeInsert = recorder(async (_quads: Quad[]) => {}); + const fetchSyncPages = recorder(async ( + _ctx: OperationContext, + _peer: string, + contextGraphId: string, + _includeSharedMemory: boolean, + phase: 'data' | 'meta', + ) => phase === 'meta' + ? pageResult(contextGraphId, phase, { nextOffset: 3, completed: false }) + : pageResult(contextGraphId, phase)); + + const summary = await runDurableSync({ + ctx, + remotePeerId: 'peer-a', + contextGraphIds: ['discarded-mixed'], + createContextGraphSyncDeadline: () => Date.now() + 60_000, + fetchSyncPages, + processDurableBatchInWorker: async () => ({ + ...durableProcessResult(), + emptyResponses: 0, + metaOnlyResponses: 1, + droppedSyncControlTriples: 1, + droppedNonIriSubjectTriples: 2, + totalFetchedMetaQuads: 3, + }), + storeInsert, + deleteCheckpoint, + setCheckpoint, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(summary.metaOnlyResponses).toBe(1); + expect(storeInsert.calls).toEqual([]); + expect(deleteCheckpoint.calls).toEqual([]); + expect(setCheckpoint.calls).toEqual([['discarded-mixed:meta', 3]]); + }); + it('deletes only the durable meta checkpoint after completing metadata-only responses', async () => { const metaQuad = quad('meta-only-complete-meta'); const storeInsert = recorder(async (_quads: Quad[]) => {}); From 6020629dfcea807aa3827ea1cb1f7724151dcd4a Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 15:51:10 +0200 Subject: [PATCH 4/9] fix(sync): stop non-IRI _meta subjects from authenticating durable data (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the otReviewAgent review of PR #1936 (two 🔴). 🔴 Data-integrity: the #1921 admission-only guard dropped non-IRI metadata from PERSISTENCE but not from VERIFICATION. indexIntegrityMetadata added any subject bearing dkg:merkleRoot to merkleSubjects with no IRI check, so a peer's blank-node/literal subject became a verification candidate, self-consistently authenticated its bound DATA (the claimed root is peer-supplied, not on-chain-anchored), and got that data admitted — after which the selector guard dropped only the metadata, leaving orphaned, peer-forged data in the store (an injection vector). Empirically: a blank-node legacy envelope over matching data yielded dataIndexes=[0], metaIndexes=[]. Fix (candidate-gate): skip non-IRI subjects at the top of indexIntegrityMetadata's merkle/marker loop so they never become verification candidates and cannot authenticate data. A page bound only by a non-IRI envelope then has no verified descriptor and fails closed (rejected on a normal CG). The admission-selector guards + droppedNonIriSubjectTriples counting are kept unchanged (still load-bearing for persist-drop and meta-cursor advance). isIriMetaSubject is now null-safe: the requester's bounded-snapshot planner (planBoundedGraphScopedDurableBatch) runs indexIntegrityMetadata on raw fetched meta, which must tolerate malformed input rather than throw. Tests: - Flip the former "persisted blank-node" case to FAIL-CLOSED for BOTH a blank-node and a literal merkle-subject (rejected=1, dataIndexes=[], metaIndexes=[]); mutation-proven (removing the gate reverts to data-persists/meta-dropped and the tests fail). - Livelock-fix-intact: an acceptUnverified all-non-IRI metadata-only page is fully consumed (droppedNonIriSubjectTriples === total, not rejected) so the cursor still advances. - Worker-wire propagation (🔴#2): real processDurableBatchForWire on an all-non-IRI page reports the dropped count across selection->wire. Deferred: consolidating the requester's per-reason consumed-metadata counters into one worker-owned aggregate touches shipped sync-control checkpoint semantics -> tracked as a follow-up, kept out of this security PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync/durable-integrity.ts | 18 +++- .../sync-control-metadata-admission.test.ts | 93 ++++++++++++------- .../test/sync-durable-worker-wire.test.ts | 30 ++++++ 3 files changed, 107 insertions(+), 34 deletions(-) diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index a889e31e46..be74a6b889 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -491,6 +491,16 @@ function indexIntegrityMetadata( const merkleSubjects = new Set(); const markerSubjects = new Set(); for (const quad of metaQuads) { + // #1921 — a non-IRI `_meta` subject must never become a verification + // candidate. Without this, a peer's blank-node/literal subject bearing + // `dkg:merkleRoot` would enter `merkleSubjects`, self-consistently + // authenticate its bound DATA (the claimed root is peer-supplied, not + // on-chain-anchored), and get that data admitted — after which the + // admission-selector guard drops only the METADATA, leaving orphaned, + // peer-forged data in the store. Gating candidacy here (not only at + // admission) closes that path; the selector guards still handle the + // persist-drop + consumed-row counting for descriptive rows. + if (!isIriMetaSubject(quad.subject)) continue; if (quad.predicate === MERKLE_ROOT) { merkleSubjects.add(quad.subject); } @@ -1452,7 +1462,13 @@ function stripLiteral(raw: string): string { * and read agree on the contract. */ function isIriMetaSubject(term: string): boolean { - return term.length > 0 && !term.startsWith('_:') && !term.startsWith('"'); + // Defensive on `term`: besides the admission selectors, this now runs inside + // `indexIntegrityMetadata`, which the requester's bounded-snapshot planner + // (`planBoundedGraphScopedDurableBatch`) invokes on RAW fetched meta before + // verification. A conforming quad always carries a non-empty string subject; + // tolerate a malformed one (treat as non-IRI → skip) instead of throwing. + return typeof term === 'string' && term.length > 0 + && !term.startsWith('_:') && !term.startsWith('"'); } function findLegacyRootOwner(subject: string, roots: ReadonlySet): string | undefined { diff --git a/packages/agent/test/sync-control-metadata-admission.test.ts b/packages/agent/test/sync-control-metadata-admission.test.ts index e67c8a89be..3f10e74b71 100644 --- a/packages/agent/test/sync-control-metadata-admission.test.ts +++ b/packages/agent/test/sync-control-metadata-admission.test.ts @@ -267,15 +267,14 @@ describe('durable sync control metadata admission', () => { // BOTH selector loops, so it fires on every guarded call site. const IRI_SUBJECT = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/50'; - it('drops a forged-integrity blank-node _meta subject on the system-override path (#1921)', () => { - // The real attack vector: a blank node bearing an INTEGRITY predicate. - // `indexIntegrityMetadata` adds ANY merkleRoot subject to `merkleSubjects` - // with no IRI check, so this reaches the merkle/marker admission machinery; - // legacy verification rejects it (no rootEntity), routing to the system - // override where every non-control row is otherwise admitted. Mirrors the - // IRI-subject case "does not let the system-graph override authenticate - // malformed legacy controls" (which KEEPS the merkleRoot) — the only - // difference is the blank-node subject, which #1921 now drops. + it('drops a forged-integrity blank-node merkleRoot subject without persisting it (#1921)', () => { + // A blank node bearing an INTEGRITY predicate. The #1921 candidate-gate in + // `indexIntegrityMetadata` excludes it from `merkleSubjects`, so it never + // becomes a verification candidate; with no verified descriptor the batch + // takes the descriptor-less admission path, where the selector guard drops + // the blank-node rows (persist-drop + count) and keeps the conforming IRI + // row. Contrast the IRI-subject case "does not let the system-graph override + // authenticate malformed legacy controls" (which KEEPS the merkleRoot). const injected = '_:injected'; const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"legit"'); const meta = [ @@ -292,30 +291,36 @@ describe('durable sync control metadata admission', () => { expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); - it('drops a blank-node _meta subject before the verified merkle/marker admission branch (#1921)', () => { - // A blank node bearing a FULLY VERIFIABLE legacy envelope (merkleRoot + - // rootEntity over matching data). Verification keys on the rootEntity, not - // the KC UAL, so the blank-node subject verifies and would otherwise reach - // `selectAdmittedMetadataIndexes`' merkle/marker branch and be persisted - // (compare "preserves batch controls bound to a verified legacy envelope"). - // The loop-top guard drops it first. - const root = 'urn:legacy:root'; - const injected = '_:injected'; - const data = [quad(root, 'urn:example:value', '"legacy"', CONTEXT_GRAPH)]; - const merkleRoot = `"${toHex(computeFlatKCRootV10(data, []))}"`; - const meta = [ - quad(injected, `${DKG}merkleRoot`, merkleRoot), - quad(injected, `${DKG}rootEntity`, root), - quad(injected, `${DKG}batchId`, integer(41n)), - ]; - - const selection = selectVerifiedDurableSyncQuads(data, meta, false); - - expect(selection.metaIndexes).toEqual([]); - expect(selection.droppedNonIriSubjectTriples).toBe(3); - expect(selection.rejected).toBe(0); - expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); - }); + for (const [label, injected] of [ + ['blank-node', '_:injected'], + ['literal', '"forged-subject"'], + ] as const) { + it(`does not let a ${label} _meta subject with dkg:merkleRoot authenticate its data (#1921)`, () => { + // #1921 root fix (candidate-gate): a non-IRI subject bearing dkg:merkleRoot + // must never become a verification candidate. Otherwise it self-consistently + // authenticates its bound DATA (the claimed root is peer-supplied, not + // on-chain-anchored), the data is admitted, and only the METADATA is later + // dropped — persisting orphaned, peer-forged data. With the gate the batch + // fails closed: no verified descriptor, so the data is rejected and NOT + // selected. Distinct from the persist-drop/cursor tests above. + const root = 'urn:legacy:root'; + const data = [quad(root, 'urn:example:value', '"legacy"', CONTEXT_GRAPH)]; + const merkleRoot = `"${toHex(computeFlatKCRootV10(data, []))}"`; + const meta = [ + quad(injected, `${DKG}merkleRoot`, merkleRoot), + quad(injected, `${DKG}rootEntity`, root), + quad(injected, `${DKG}batchId`, integer(41n)), + ]; + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + // Fail closed: the non-IRI envelope authenticates nothing, so neither the + // data nor its metadata is persisted. + expect(selection.rejected).toBe(1); + expect(selection.dataIndexes).toEqual([]); + expect(selection.metaIndexes).toEqual([]); + }); + } it('drops a non-IRI _meta subject on a descriptor-less system page (#1921)', () => { // The :401 no-merkle/no-marker branch: `selectAdmittedMetadataIndexes` runs @@ -335,6 +340,28 @@ describe('durable sync control metadata admission', () => { expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); + it('counts an all-non-IRI system-CG metadata-only page as fully consumed so the cursor advances (#1921)', () => { + // Livelock-fix-intact guard: after the candidate-gate, an acceptUnverified + // (system-CG) metadata-only page consisting ENTIRELY of non-IRI subjects — + // even a forged dkg:merkleRoot — is neither rejected nor pinned. The forged + // merkle subject is excluded from candidacy, so no verified descriptor + // exists; with no data the batch is not rejected, and every row is dropped + // and COUNTED (droppedNonIriSubjectTriples === total). That equality is what + // lets the requester advance the meta cursor instead of re-fetching forever. + const meta = [ + quad('_:injected', `${DKG}merkleRoot`, `"${'00'.repeat(32)}"`), + quad('_:injected', `${DKG}status`, '"forged"'), + quad('"literal-subject"', `${DKG}status`, '"drop-too"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.rejected).toBe(0); + expect(selection.metaIndexes).toEqual([]); + expect(selection.droppedNonIriSubjectTriples).toBe(meta.length); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + it('keeps a conforming IRI _meta subject untouched (no #1921 drop)', () => { const scope = createGraphKnowledgeAssetScope(UAL, '1'); const assertionGraph = knowledgeAssetLayerGraphUri( diff --git a/packages/agent/test/sync-durable-worker-wire.test.ts b/packages/agent/test/sync-durable-worker-wire.test.ts index 31036e5781..0d832d55e7 100644 --- a/packages/agent/test/sync-durable-worker-wire.test.ts +++ b/packages/agent/test/sync-durable-worker-wire.test.ts @@ -337,4 +337,34 @@ describe('durable sync worker result transport', () => { expect(wireResult.metaOnlyResponses).toBe(1); expect(wireResult.rejectedKcs).toBe(0); }); + + it('reports discarded-only non-IRI meta subjects across the worker wire (#1921)', () => { + // Exercises the REAL selection -> processDurableBatchForWire path so a + // regression that drops droppedNonIriSubjectTriples between the selector and + // the wire result is caught (the requester-progress unit tests inject the + // count directly and cannot see this). A metadata-only page of purely + // non-IRI subjects must report the full dropped count so the requester can + // advance the meta cursor instead of pinning. + const meta: Quad[] = [ + { + subject: '_:injected', + predicate: 'http://dkg.io/ontology/status', + object: '"drop"', + graph: META_GRAPH, + }, + { + subject: '"literal-subject"', + predicate: 'http://dkg.io/ontology/status', + object: '"drop-too"', + graph: META_GRAPH, + }, + ]; + + const wireResult = processDurableBatchForWire([], meta, false); + + expect(wireResult.verifiedMetaIndexes).toEqual([]); + expect(wireResult.droppedNonIriSubjectTriples).toBe(meta.length); + expect(wireResult.metaOnlyResponses).toBe(1); + expect(wireResult.rejectedKcs).toBe(0); + }); }); From f6aec0f423e3d97a872985042bbf47a2352afaa6 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 16:14:18 +0200 Subject: [PATCH 5/9] refactor(sync): consolidate durable meta-checkpoint accounting into one worker-owned count (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the otReviewAgent review of PR #1936 (🟡: requester progress depended on per-reason verifier discard counters). The requester's discardedOnlyMetadataResponse summed reason-specific counters (droppedSyncControlTriples + droppedNonIriSubjectTriples) to decide whether a fully-discarded metadata-only page still advances the meta cursor. That coupled checkpoint orchestration to verifier discard policy, so each new deliberate-drop category would need another optional field threaded through the worker result plus another addend here. Consolidate: the WORKER (processDurableBatch) now emits a single reason-agnostic consumedUnpersistedMetaTriples = droppedSyncControlTriples + droppedNonIriSubjectTriples on DurableBatchProcessResult, and the requester's checkpoint predicate compares that one field to totalFetchedMetaQuads. The per-reason counters are kept as verifier-side diagnostics only and removed from the requester's structural contract. The already-shipped sync-control semantics are preserved bit-for-bit: for a pure-sync-control page nonIri=0 so the aggregate equals droppedSyncControlTriples exactly. Tests: - Worker-level: consumedUnpersistedMetaTriples === droppedSyncControlTriples + droppedNonIriSubjectTriples for pure-sync-control, all-non-IRI, and mixed pages. - Requester cursor-advance for pure-sync-control (regression guard for the shipped path), all-non-IRI, and mixed pages, all via the single aggregate; pin case when the aggregate < total. - Mutation-proven: neutralizing the worker sum breaks the all-non-IRI + mixed aggregate tests; neutralizing the requester's aggregate read breaks all three cursor-advance tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync-verify-worker-impl.ts | 6 ++++ packages/agent/src/sync-verify-worker.ts | 17 +++++++--- .../agent/src/sync/requester/durable-sync.ts | 27 ++++++++-------- .../test/sync-durable-worker-wire.test.ts | 32 +++++++++++++++++++ .../test/sync-requester-progress.test.ts | 29 ++++++++--------- 5 files changed, 77 insertions(+), 34 deletions(-) diff --git a/packages/agent/src/sync-verify-worker-impl.ts b/packages/agent/src/sync-verify-worker-impl.ts index f12ec2fa7a..e007ce0222 100644 --- a/packages/agent/src/sync-verify-worker-impl.ts +++ b/packages/agent/src/sync-verify-worker-impl.ts @@ -308,6 +308,7 @@ function processDurableBatch( verifiedGraphScopedDataGraphs: [], droppedSyncControlTriples: 0, droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedPrivateOnlyResponses: 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -332,6 +333,7 @@ function processDurableBatch( verifiedGraphScopedDataGraphs: [], droppedSyncControlTriples: 0, droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedPrivateOnlyResponses: 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -387,6 +389,8 @@ function processDurableBatch( verifiedGraphScopedDataGraphs: verifiedSelection.verifiedGraphScopedDataGraphs, droppedSyncControlTriples: verifiedSelection.droppedSyncControlTriples, droppedNonIriSubjectTriples: verifiedSelection.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: + verifiedSelection.droppedSyncControlTriples + verifiedSelection.droppedNonIriSubjectTriples, verifiedPrivateOnlyResponses: verifiedFullyPrivateResponse ? 1 : 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -427,6 +431,7 @@ export function processDurableBatchForWire( verifiedGraphScopedDataGraphs, droppedSyncControlTriples, droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples, verifiedPrivateOnlyResponses, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -444,6 +449,7 @@ export function processDurableBatchForWire( verifiedGraphScopedDataGraphs, droppedSyncControlTriples, droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples, verifiedPrivateOnlyResponses, totalFetchedDataQuads, totalFetchedMetaQuads, diff --git a/packages/agent/src/sync-verify-worker.ts b/packages/agent/src/sync-verify-worker.ts index 07cf172be9..cf8e7ff705 100644 --- a/packages/agent/src/sync-verify-worker.ts +++ b/packages/agent/src/sync-verify-worker.ts @@ -46,16 +46,23 @@ export interface DurableBatchProcessResult { verifiedData: Quad[]; verifiedMeta: Quad[]; verifiedGraphScopedDataGraphs: string[]; - /** Metadata controls deliberately consumed after failing authentication. */ + /** Metadata controls deliberately consumed after failing authentication (diagnostic). */ droppedSyncControlTriples: number; /** * Non-IRI (blank-node/literal) `_meta` subject rows deliberately dropped at - * ingest (#1921). Surfaced beside {@link droppedSyncControlTriples} so the - * requester counts them as consumed metadata for meta-checkpoint advancement - * (a metadata-only page discarded entirely by the guard must still advance - * the cursor, not pin durable sync on the same page). + * ingest (#1921) — a verifier-side diagnostic count. */ droppedNonIriSubjectTriples: number; + /** + * Reason-agnostic aggregate of meta rows the verifier deliberately CONSUMED + * but did not persist (unverified sync controls + non-IRI subjects). This is + * the single count the requester uses to decide whether a fully-discarded + * metadata-only page still advances the meta checkpoint (rather than pinning + * durable sync on the same page). Keeping the per-reason counts above as + * diagnostics only keeps checkpoint orchestration decoupled from verifier + * discard policy. + */ + consumedUnpersistedMetaTriples: number; /** Clean batches containing verified V2 assets with no public assertion triples. */ verifiedPrivateOnlyResponses: number; totalFetchedDataQuads: number; diff --git a/packages/agent/src/sync/requester/durable-sync.ts b/packages/agent/src/sync/requester/durable-sync.ts index 7a229234b7..aa78a1f772 100644 --- a/packages/agent/src/sync/requester/durable-sync.ts +++ b/packages/agent/src/sync/requester/durable-sync.ts @@ -117,8 +117,8 @@ interface DurableSyncContext { verifiedData: Quad[]; verifiedMeta: Quad[]; verifiedGraphScopedDataGraphs?: string[]; - droppedSyncControlTriples?: number; - droppedNonIriSubjectTriples?: number; + /** Worker-owned aggregate of meta rows deliberately consumed but not persisted. */ + consumedUnpersistedMetaTriples?: number; totalFetchedDataQuads: number; totalFetchedMetaQuads: number; rejectedKcs: number; @@ -473,21 +473,20 @@ export async function runDurableSync( }; const metadataOnlyResponse = processed.metaOnlyResponses > 0; - const droppedSyncControlTriples = processed.droppedSyncControlTriples ?? 0; - // Rows the verifier deliberately consumed but did not persist: unverified - // sync controls (existing) plus non-IRI `_meta` subjects dropped at ingest - // (#1921). A metadata-only page discarded ENTIRELY by these guards carries - // no verifiedMeta, so — like the all-controls case — the meta cursor only - // advances if we still count the page as consumed. Summing both kinds also - // covers a MIXED all-discarded page (some controls + some non-IRI), which - // otherwise pins because neither count alone equals the fetched total. - const deliberatelyDroppedMeta = - droppedSyncControlTriples + (processed.droppedNonIriSubjectTriples ?? 0); + // The worker reports, as ONE reason-agnostic count, how many fetched meta + // rows the verifier deliberately consumed but did NOT persist — unverified + // sync controls plus non-IRI `_meta` subjects (#1921). A metadata-only page + // discarded ENTIRELY this way carries no verifiedMeta, so the meta cursor + // must still advance or durable sync pins on the same page. Depending on the + // aggregate (not per-reason counters) keeps checkpoint orchestration + // decoupled from verifier discard policy; the per-reason counts remain as + // verifier-side diagnostics only. + const consumedUnpersistedMetaTriples = processed.consumedUnpersistedMetaTriples ?? 0; const discardedOnlyMetadataResponse = metadataOnlyResponse && processed.verifiedData.length === 0 && processed.verifiedMeta.length === 0 - && deliberatelyDroppedMeta > 0 - && deliberatelyDroppedMeta === processed.totalFetchedMetaQuads; + && consumedUnpersistedMetaTriples > 0 + && consumedUnpersistedMetaTriples === processed.totalFetchedMetaQuads; const updateMetaCheckpoint = batchVerifiedCleanly && processed.dataRejectedMissingMeta === 0 && ( diff --git a/packages/agent/test/sync-durable-worker-wire.test.ts b/packages/agent/test/sync-durable-worker-wire.test.ts index 0d832d55e7..59afbf7986 100644 --- a/packages/agent/test/sync-durable-worker-wire.test.ts +++ b/packages/agent/test/sync-durable-worker-wire.test.ts @@ -334,6 +334,9 @@ describe('durable sync worker result transport', () => { expect(wireResult.verifiedMetaIndexes).toEqual([]); expect(wireResult.droppedSyncControlTriples).toBe(meta.length); + // Pure-sync-control page: the requester-facing aggregate equals the + // sync-control diagnostic count, preserving the shipped checkpoint semantics. + expect(wireResult.consumedUnpersistedMetaTriples).toBe(meta.length); expect(wireResult.metaOnlyResponses).toBe(1); expect(wireResult.rejectedKcs).toBe(0); }); @@ -364,6 +367,35 @@ describe('durable sync worker result transport', () => { expect(wireResult.verifiedMetaIndexes).toEqual([]); expect(wireResult.droppedNonIriSubjectTriples).toBe(meta.length); + // All-non-IRI page: the aggregate equals the non-IRI diagnostic count. + expect(wireResult.consumedUnpersistedMetaTriples).toBe(meta.length); + expect(wireResult.metaOnlyResponses).toBe(1); + expect(wireResult.rejectedKcs).toBe(0); + }); + + it('aggregates mixed control + non-IRI drops into consumedUnpersistedMetaTriples (#1921)', () => { + // Worker-level guard for the consolidated checkpoint contract: the single + // requester-facing aggregate equals the SUM of the per-reason diagnostic + // counts for a page mixing unverified sync controls and non-IRI subjects. + // Neutralizing the worker sum breaks this; the per-reason counts remain as + // diagnostics. + const control = 'urn:orphaned:sync-control'; + const meta: Quad[] = [ + { subject: control, predicate: 'http://dkg.io/ontology/batchId', object: '"999"^^', graph: META_GRAPH }, + { subject: control, predicate: 'http://dkg.io/ontology/assertionGraph', object: 'urn:attacker:graph', graph: META_GRAPH }, + { subject: '_:injected', predicate: 'http://dkg.io/ontology/status', object: '"drop"', graph: META_GRAPH }, + { subject: '"literal-subject"', predicate: 'http://dkg.io/ontology/status', object: '"drop-too"', graph: META_GRAPH }, + ]; + + const wireResult = processDurableBatchForWire([], meta, false); + + expect(wireResult.verifiedMetaIndexes).toEqual([]); + expect(wireResult.droppedSyncControlTriples).toBe(2); + expect(wireResult.droppedNonIriSubjectTriples).toBe(2); + expect(wireResult.consumedUnpersistedMetaTriples).toBe( + wireResult.droppedSyncControlTriples + wireResult.droppedNonIriSubjectTriples, + ); + expect(wireResult.consumedUnpersistedMetaTriples).toBe(meta.length); expect(wireResult.metaOnlyResponses).toBe(1); expect(wireResult.rejectedKcs).toBe(0); }); diff --git a/packages/agent/test/sync-requester-progress.test.ts b/packages/agent/test/sync-requester-progress.test.ts index a88e6f19d1..5054b4e4c1 100644 --- a/packages/agent/test/sync-requester-progress.test.ts +++ b/packages/agent/test/sync-requester-progress.test.ts @@ -57,7 +57,7 @@ function durableProcessResult() { return { verifiedData: [] as Quad[], verifiedMeta: [] as Quad[], - droppedSyncControlTriples: 0, + consumedUnpersistedMetaTriples: 0, totalFetchedDataQuads: 0, totalFetchedMetaQuads: 0, rejectedKcs: 0, @@ -515,7 +515,9 @@ describe('sync requester progress accounting', () => { ...durableProcessResult(), emptyResponses: 0, metaOnlyResponses: 1, - droppedSyncControlTriples: 3, + // Pure-sync-control page: worker aggregates 3 discarded controls into + // consumedUnpersistedMetaTriples. Regression guard for the shipped path. + consumedUnpersistedMetaTriples: 3, totalFetchedMetaQuads: 3, }), storeInsert, @@ -556,7 +558,7 @@ describe('sync requester progress accounting', () => { ...durableProcessResult(), emptyResponses: 0, metaOnlyResponses: 1, - droppedSyncControlTriples: 2, + consumedUnpersistedMetaTriples: 2, totalFetchedMetaQuads: 3, }), storeInsert: async () => {}, @@ -572,11 +574,10 @@ describe('sync requester progress accounting', () => { }); it('advances the meta cursor when the whole page is non-IRI subjects dropped at ingest (#1921)', async () => { - // A metadata-only page consisting solely of non-IRI `_meta` rows is fully - // discarded by the #1921 guard: verifiedMeta=[] and droppedSyncControlTriples=0, - // but droppedNonIriSubjectTriples === totalFetchedMetaQuads. The meta cursor - // must still advance (the rows were deliberately consumed) or durable sync - // pins on the same poisoned page. Fails before the propagate fix. + // All-non-IRI metadata-only page: the worker aggregates every dropped row + // into consumedUnpersistedMetaTriples === totalFetchedMetaQuads and no meta + // is persisted. The requester must still advance the meta cursor (the rows + // were deliberately consumed) or durable sync pins on the same poisoned page. const setCheckpoint = recorder((_key: string, _offset: number) => {}); const deleteCheckpoint = recorder((_key: string) => {}); const storeInsert = recorder(async (_quads: Quad[]) => {}); @@ -600,8 +601,7 @@ describe('sync requester progress accounting', () => { ...durableProcessResult(), emptyResponses: 0, metaOnlyResponses: 1, - droppedSyncControlTriples: 0, - droppedNonIriSubjectTriples: 3, + consumedUnpersistedMetaTriples: 3, totalFetchedMetaQuads: 3, }), storeInsert, @@ -620,9 +620,9 @@ describe('sync requester progress accounting', () => { }); it('advances the meta cursor when a mixed page is fully discarded by controls + non-IRI drops (#1921)', async () => { - // A mixed all-discarded page (some unverified controls + some non-IRI rows) - // also pins today because neither count alone equals the fetched total. The - // summed deliberate-drop check advances the cursor. Fails before the fix. + // A mixed all-discarded page (some unverified controls + some non-IRI rows): + // the worker sums both reasons into consumedUnpersistedMetaTriples === the + // fetched total, so the requester advances the cursor rather than pinning. const setCheckpoint = recorder((_key: string, _offset: number) => {}); const deleteCheckpoint = recorder((_key: string) => {}); const storeInsert = recorder(async (_quads: Quad[]) => {}); @@ -646,8 +646,7 @@ describe('sync requester progress accounting', () => { ...durableProcessResult(), emptyResponses: 0, metaOnlyResponses: 1, - droppedSyncControlTriples: 1, - droppedNonIriSubjectTriples: 2, + consumedUnpersistedMetaTriples: 3, totalFetchedMetaQuads: 3, }), storeInsert, From c42c48818f22db4cb3995c319d32fa609bccac80 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 16:37:00 +0200 Subject: [PATCH 6/9] fix(sync): sanitize non-IRI _meta out of durable verification, not just admission (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the otReviewAgent 14:03 review of PR #1936 (a 🔴 nested on the resolved durable-integrity thread) + follow-up 🟡s. 🔴 Non-IRI metadata could still POISON verification. The candidate-gate stopped non-IRI subjects from becoming merkle/marker candidates, but metaBySubject was still built from ALL rows, and readIntegrityMetadata scans it for PART_OF (line ~604) — so a peer's `_:bad dkg:partOf ""` row falsely invalidated the valid graph-scoped UAL → fatalUnscopedFailure → a VALID batch was rejected → durable sync pinned (a poison/DoS vector, opposite direction to the earlier authenticate-then-drop hole). Reproduced: valid descriptor + one non-IRI partOf row → rejected=1, dataIndexes=[], metaIndexes=[]. Fix (Option B, boundary sanitize): filter metaQuads to IRI-only ONCE at each verification entry and feed that to indexIntegrityMetadata + verifyLegacyCandidates, so non-IRI subjects can neither authenticate data (never a candidate) nor poison verification (never in metaBySubject or the raw legacy scan). This is applied in BOTH selectVerifiedDurableSyncQuads AND planBoundedGraphScopedDurableBatch — both call readIntegrityMetadata and were vulnerable. Admission (selectVerifiedQuads / the no-descriptor selectAdmittedMetadataIndexes) deliberately stays on the ORIGINAL metaQuads: the selectors still drop + count non-IRI rows (persist-drop and meta-cursor advance) and index into the original array. The verification outcome is subject-keyed, so no positional index remap is needed. The now-subsumed candidate-gate inside indexIntegrityMetadata is removed (single boundary invariant; addresses the "centralize the drop" 🟡). Also (net-new 🟡): make consumedUnpersistedMetaTriples REQUIRED (non-optional) on the requester's processed contract and drop the `?? 0` — it is the single checkpoint-advance signal, so every producer must set it (tsc-enforced) rather than silently reading 0 and pinning the cursor. Tests: - Valid graph-scoped descriptor + `_:bad dkg:partOf ` → asset STILL verifies (rejected=0, dataIndexes=[0], full descriptor persisted), bad row dropped + counted; mutation-proven (neutralize the sanitize → rejected=1). - Direct selectSystemOverrideMetadataIndexes non-IRI test (rejected IRI candidate forces the accept-unverified override path + non-IRI rows → dropped + counted). - Updated the since-threading mock for the now-required aggregate. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync/durable-integrity.ts | 49 ++++++++++------- .../agent/src/sync/requester/durable-sync.ts | 11 ++-- .../test/durable-sync-since-threading.test.ts | 1 + .../sync-control-metadata-admission.test.ts | 54 +++++++++++++++++++ 4 files changed, 94 insertions(+), 21 deletions(-) diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index be74a6b889..c0799d6093 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -173,7 +173,13 @@ export function planBoundedGraphScopedDurableBatch( || metaQuads.length === 0 ) return null; - const metadata = indexIntegrityMetadata(dataQuads, metaQuads); + // #1921 — verification must never see non-IRI `_meta` subjects: a peer's + // `_:bad dkg:partOf ""` row would otherwise be scanned by + // readIntegrityMetadata and falsely invalidate the valid graph-scoped UAL. + // Sanitize before indexing. The bounded planner returns only data offsets, + // so dropping non-IRI meta rows here is loss-free. + const iriMetaQuads = metaQuads.filter((quad) => isIriMetaSubject(quad.subject)); + const metadata = indexIntegrityMetadata(dataQuads, iriMetaQuads); const parsed = readIntegrityMetadata(metadata, false); if ( parsed.fatalUnscopedFailure @@ -385,7 +391,17 @@ export function selectVerifiedDurableSyncQuads( }; } - const metadata = indexIntegrityMetadata(dataQuads, metaQuads); + // #1921 — sanitize the verification inputs ONCE at this boundary so a non-IRI + // `_meta` subject can neither authenticate data (it never becomes a candidate) + // NOR poison verification (the readIntegrityMetadata PART_OF scan and + // verifyLegacyCandidates' raw scan never see it, so a `_:bad dkg:partOf + // ""` row cannot falsely invalidate a valid batch). Admission below + // deliberately runs on the ORIGINAL metaQuads: the selectors still drop+count + // non-IRI rows (persist-drop + meta-cursor advance) and index into the + // original array. The verification outcome is subject-keyed, so no positional + // remap is needed across the sanitized/original split. + const iriMetaQuads = metaQuads.filter((quad) => isIriMetaSubject(quad.subject)); + const metadata = indexIntegrityMetadata(dataQuads, iriMetaQuads); if (metadata.merkleSubjects.size === 0 && metadata.markerSubjects.size === 0) { if (!acceptUnverified && dataQuads.length > 0) { logs.push({ @@ -434,7 +450,7 @@ export function selectVerifiedDurableSyncQuads( ); const legacy = verifyLegacyCandidates( dataQuads, - metaQuads, + iriMetaQuads, metadata, parsed.candidates, acceptUnverified, @@ -490,17 +506,14 @@ function indexIntegrityMetadata( const merkleSubjects = new Set(); const markerSubjects = new Set(); + // #1921 — callers pass IRI-only `_meta` quads: both selectVerifiedDurableSyncQuads + // and planBoundedGraphScopedDurableBatch sanitize non-IRI subjects at their + // boundary before indexing. So no blank-node/literal subject reaches candidacy + // OR the metaBySubject-based verification scans (readIntegrityMetadata's PART_OF + // scan, parseGraphScopedDescriptor), meaning a non-IRI subject can neither + // authenticate data nor poison a valid batch. Admission (the selectors) runs on + // the ORIGINAL metaQuads and is where non-IRI rows are dropped + counted. for (const quad of metaQuads) { - // #1921 — a non-IRI `_meta` subject must never become a verification - // candidate. Without this, a peer's blank-node/literal subject bearing - // `dkg:merkleRoot` would enter `merkleSubjects`, self-consistently - // authenticate its bound DATA (the claimed root is peer-supplied, not - // on-chain-anchored), and get that data admitted — after which the - // admission-selector guard drops only the METADATA, leaving orphaned, - // peer-forged data in the store. Gating candidacy here (not only at - // admission) closes that path; the selector guards still handle the - // persist-drop + consumed-row counting for descriptive rows. - if (!isIriMetaSubject(quad.subject)) continue; if (quad.predicate === MERKLE_ROOT) { merkleSubjects.add(quad.subject); } @@ -1462,11 +1475,11 @@ function stripLiteral(raw: string): string { * and read agree on the contract. */ function isIriMetaSubject(term: string): boolean { - // Defensive on `term`: besides the admission selectors, this now runs inside - // `indexIntegrityMetadata`, which the requester's bounded-snapshot planner - // (`planBoundedGraphScopedDurableBatch`) invokes on RAW fetched meta before - // verification. A conforming quad always carries a non-empty string subject; - // tolerate a malformed one (treat as non-IRI → skip) instead of throwing. + // Defensive on `term`: this runs at the verification-input boundary filters + // (selectVerifiedDurableSyncQuads and planBoundedGraphScopedDurableBatch, both + // on RAW fetched meta) and in the admission selectors. A conforming quad always + // carries a non-empty string subject; tolerate a malformed one (treat as + // non-IRI → drop) instead of throwing. return typeof term === 'string' && term.length > 0 && !term.startsWith('_:') && !term.startsWith('"'); } diff --git a/packages/agent/src/sync/requester/durable-sync.ts b/packages/agent/src/sync/requester/durable-sync.ts index aa78a1f772..9705ac3aeb 100644 --- a/packages/agent/src/sync/requester/durable-sync.ts +++ b/packages/agent/src/sync/requester/durable-sync.ts @@ -117,8 +117,13 @@ interface DurableSyncContext { verifiedData: Quad[]; verifiedMeta: Quad[]; verifiedGraphScopedDataGraphs?: string[]; - /** Worker-owned aggregate of meta rows deliberately consumed but not persisted. */ - consumedUnpersistedMetaTriples?: number; + /** + * Worker-owned aggregate of meta rows deliberately consumed but not + * persisted. REQUIRED (#1921): it is the single checkpoint-advance signal, + * so every producer must set it — an optional field silently reading 0 + * would let the meta cursor pin with no type error. + */ + consumedUnpersistedMetaTriples: number; totalFetchedDataQuads: number; totalFetchedMetaQuads: number; rejectedKcs: number; @@ -481,7 +486,7 @@ export async function runDurableSync( // aggregate (not per-reason counters) keeps checkpoint orchestration // decoupled from verifier discard policy; the per-reason counts remain as // verifier-side diagnostics only. - const consumedUnpersistedMetaTriples = processed.consumedUnpersistedMetaTriples ?? 0; + const consumedUnpersistedMetaTriples = processed.consumedUnpersistedMetaTriples; const discardedOnlyMetadataResponse = metadataOnlyResponse && processed.verifiedData.length === 0 && processed.verifiedMeta.length === 0 diff --git a/packages/agent/test/durable-sync-since-threading.test.ts b/packages/agent/test/durable-sync-since-threading.test.ts index 95996b2288..30d25f6919 100644 --- a/packages/agent/test/durable-sync-since-threading.test.ts +++ b/packages/agent/test/durable-sync-since-threading.test.ts @@ -171,6 +171,7 @@ function makeContext(options: { return { verifiedData, verifiedMeta, + consumedUnpersistedMetaTriples: 0, totalFetchedDataQuads: options.processResult?.totalFetchedDataQuads ?? dataQuads.length, totalFetchedMetaQuads: options.processResult?.totalFetchedMetaQuads ?? metaQuads.length, rejectedKcs: 0, diff --git a/packages/agent/test/sync-control-metadata-admission.test.ts b/packages/agent/test/sync-control-metadata-admission.test.ts index 3f10e74b71..50d857ca3b 100644 --- a/packages/agent/test/sync-control-metadata-admission.test.ts +++ b/packages/agent/test/sync-control-metadata-admission.test.ts @@ -406,4 +406,58 @@ describe('durable sync control metadata admission', () => { expect(selection.droppedNonIriSubjectTriples).toBe(1); expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); + + it('verifies a graph-scoped asset even when a non-IRI dkg:partOf row is present (#1921)', () => { + // A peer can attach `_:bad dkg:partOf ""` beside a valid + // graph-scoped descriptor. Before the #1921 verification-input sanitize, + // readIntegrityMetadata's PART_OF scan over metaBySubject saw that non-IRI + // row and falsely invalidated the valid UAL → the whole batch was rejected + // → durable sync pinned (a poison/DoS vector). The boundary sanitize keeps + // non-IRI subjects out of verification, so the asset still verifies while + // the bad row is dropped + counted (never persisted). + const scope = createGraphKnowledgeAssetScope(UAL, '1'); + const assertionGraph = knowledgeAssetLayerGraphUri(CONTEXT_GRAPH_ID, MemoryLayer.VerifiableMemory, scope); + const data = [quad('urn:verified:entity', 'urn:example:value', '"verified"', assertionGraph)]; + const verified = generateGraphKnowledgeAssetMetadata({ + ual: UAL, + contextGraphId: CONTEXT_GRAPH_ID, + merkleRoot: computeFlatKCRootV10(data, []), + publisherPeerId: 'verified-publisher', + accessPolicy: 'public', + timestamp: new Date(0), + assertionVersion: '1', + publicTripleCount: data.length, + assertionGraph, + }, { status: 'tentative' }); + const poison = quad('_:bad', `${DKG}partOf`, `"${UAL}"`); + const meta = [...verified, poison]; + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + expect(selection.rejected).toBe(0); + expect(selection.dataIndexes).toEqual([0]); + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual(verified); + expect(selection.droppedNonIriSubjectTriples).toBe(1); + }); + + it('drops a non-IRI _meta subject on the system-override path (#1921)', () => { + // Directly exercises selectSystemOverrideMetadataIndexes: an IRI merkle + // subject that fails legacy verification (no rootEntity) forces the + // accept-unverified system-override path (rejected>0 && acceptUnverified), + // where every non-control row is otherwise admitted. The non-IRI rows must + // be dropped + counted on THIS branch, not persisted. + const iriSubject = 'urn:system:malformed-legacy'; + const keptMerkle = quad(iriSubject, `${DKG}merkleRoot`, `"${'00'.repeat(32)}"`); + const meta = [ + keptMerkle, + quad('_:injected', `${DKG}status`, '"drop"'), + quad('"literal-subject"', `${DKG}label`, '"drop-too"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptMerkle]); + expect(selection.droppedNonIriSubjectTriples).toBe(2); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); }); From 9cc42e397496b7e54a076172409127899811ce79 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 16:43:58 +0200 Subject: [PATCH 7/9] docs(sync): make indexIntegrityMetadata IRI-precondition explicit; fix stale gate comment (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment-only. Two follow-ups to the Option B boundary-sanitize: - indexIntegrityMetadata now documents an explicit PRECONDITION — callers MUST pass IRI-sanitized meta (selectVerifiedDurableSyncQuads + planBounded do; a new caller must too) — since the internal candidate-gate was removed. - Rewrite the selectAdmittedMetadataIndexes non-IRI guard comment: it claimed indexIntegrityMetadata "adds ANY merkleRoot-bearing subject" (the bot's cited drift), which is no longer true under the boundary sanitize. The guard's real job is admission drop+count on the ORIGINAL metaQuads (persist-drop + cursor accounting); non-IRI subjects can't reach the merkle/marker branch anymore. No logic change. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync/durable-integrity.ts | 35 ++++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index c0799d6093..a607dc2337 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -506,13 +506,16 @@ function indexIntegrityMetadata( const merkleSubjects = new Set(); const markerSubjects = new Set(); - // #1921 — callers pass IRI-only `_meta` quads: both selectVerifiedDurableSyncQuads - // and planBoundedGraphScopedDurableBatch sanitize non-IRI subjects at their - // boundary before indexing. So no blank-node/literal subject reaches candidacy - // OR the metaBySubject-based verification scans (readIntegrityMetadata's PART_OF - // scan, parseGraphScopedDescriptor), meaning a non-IRI subject can neither - // authenticate data nor poison a valid batch. Admission (the selectors) runs on - // the ORIGINAL metaQuads and is where non-IRI rows are dropped + counted. + // #1921 PRECONDITION: callers MUST pass IRI-sanitized `_meta` quads. Both + // current callers do — selectVerifiedDurableSyncQuads and + // planBoundedGraphScopedDurableBatch filter non-IRI subjects at their boundary + // before indexing — and any NEW caller MUST too. This function no longer + // filters internally, so a non-IRI subject reaching here would re-enter + // candidacy AND the metaBySubject-based verification scans + // (readIntegrityMetadata's PART_OF scan, parseGraphScopedDescriptor), letting a + // peer's blank-node/literal subject authenticate data OR poison a valid batch. + // Admission (the selectors) deliberately runs on the ORIGINAL metaQuads and is + // where non-IRI rows are dropped + counted. for (const quad of metaQuads) { if (quad.predicate === MERKLE_ROOT) { merkleSubjects.add(quad.subject); @@ -1113,17 +1116,13 @@ function selectAdmittedMetadataIndexes( let droppedNonIriSubjectTriples = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; - // #1921 — a durable `_meta` subject must be a conforming IRI. Drop a - // blank-node/literal subject here, BEFORE the merkle/marker admission - // branch below: `indexIntegrityMetadata` adds ANY merkleRoot-bearing - // subject to `merkleSubjects` without validating the subject term, so a - // forged-integrity blank node would otherwise fall into that branch and be - // persisted. The ordering is load-bearing — the check must run before the - // branch, not inside it: a blank node has no stable identity, and once - // persisted the responder must serve it back, where #1916's store-paged - // subject-atomic lane only stays sound for it via Oxigraph's per-query - // relabel. Rejecting at the loop top preserves the store invariant that - // every durable `_meta` subject is a conforming IRI. + // #1921 — admission runs on the ORIGINAL metaQuads (verification already ran + // on the IRI-sanitized set at the boundary). A non-IRI descriptive row would + // otherwise reach the descriptive fall-through below and be persisted, so + // drop + count it here — that keeps it out of the store AND feeds the + // meta-cursor consumed-row count (checkpoint advance). It could not reach the + // merkle/marker branch regardless: the boundary sanitize keeps non-IRI + // subjects out of `merkleSubjects`/`markerSubjects`. if (!isIriMetaSubject(quad.subject)) { droppedNonIriSubjectTriples += 1; continue; From c12e6c36624ad961ab2fea42dd06c49e0209a65d Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 16:55:35 +0200 Subject: [PATCH 8/9] test(sync): cover the bounded-planner non-IRI partOf poison path (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the otReviewAgent :181 🟡 — Option B's sanitize was added to the second verification boundary (planBoundedGraphScopedDurableBatch) but only the main selector path had a poison regression. Add a bounded-planner test: a valid 3-graph manifest + `_:bad dkg:partOf ""` → the planner still projects the same safe prefix (safeNextOffset=8, 2 complete graphs) as the clean case, rather than returning null. Mutation-proven: removing planBounded's iriMetaQuads filter makes the plan null (poison invalidates the valid UAL) and this test fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../rootless-durable-bounded-progress.test.ts | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/agent/test/rootless-durable-bounded-progress.test.ts b/packages/agent/test/rootless-durable-bounded-progress.test.ts index c06dcfa711..9d2a9441af 100644 --- a/packages/agent/test/rootless-durable-bounded-progress.test.ts +++ b/packages/agent/test/rootless-durable-bounded-progress.test.ts @@ -148,6 +148,50 @@ describe('bounded rootless durable progress', () => { ]); }); + it('ignores a non-IRI dkg:partOf poison row and still projects the valid boundary (#1921)', () => { + // A peer can attach `_:bad dkg:partOf ""` to a valid graph-scoped + // manifest. Without the bounded planner's #1921 verification-input sanitize, + // readIntegrityMetadata's PART_OF scan would invalidate that valid UAL → + // planBoundedGraphScopedDurableBatch returns null → timed-out graph-scoped + // progress is lost (sync pins). The sanitize keeps non-IRI subjects out of + // verification, so the planner still projects the same safe prefix as the + // clean case above. Mutation check: remove the iriMetaQuads filter from + // planBoundedGraphScopedDurableBatch and this returns null. + const fixtures = orderedAssets(); + const meta = fixtures.flatMap((entry) => entry.meta); + const poison: Quad = { + subject: '_:bad', + predicate: 'http://dkg.io/ontology/partOf', + object: `"${fixtures[0]!.ual}"`, + graph: `${CONTEXT_GRAPH_URI}/_meta`, + }; + const rawData = [ + ...fixtures[0]!.payload, + ...fixtures[1]!.payload, + ...fixtures[2]!.payload.slice(0, 2), + ]; + + const plan = planBoundedGraphScopedDurableBatch( + rawData, + [...meta, poison], + 0, + rawData.length, + false, + ); + + expect(plan).not.toBeNull(); + expect(plan?.safeNextOffset).toBe(8); + expect(plan?.completedGraphCount).toBe(2); + expect(plan?.changedDataGraphs).toEqual([ + fixtures[0]!.graph, + fixtures[1]!.graph, + ]); + expect(plan?.dataQuads).toEqual([ + ...fixtures[0]!.payload, + ...fixtures[1]!.payload, + ]); + }); + it('replays one complete graph when timeout lands exactly on a boundary', () => { const fixtures = orderedAssets(); const meta = fixtures.flatMap((entry) => entry.meta); From b7710dcf1155fa2f9cd6ecb05eec2dd01742d1ff Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 23 Jul 2026 17:15:27 +0200 Subject: [PATCH 9/9] refactor(sync): let the verifier own the consumed-metadata checkpoint aggregate (#1921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the otReviewAgent :392 🟡 — consumedUnpersistedMetaTriples (the single signal the requester uses for meta-checkpoint advance) was computed one layer up in the worker by re-summing the verifier's diagnostic counters, so the worker had to know which discard reasons count toward requester progress. Move ownership to the verifier that classifies the drops: add consumedUnpersistedMetaTriples (REQUIRED) to DurableIntegritySelection, set at every return site as droppedSyncControlTriples + droppedNonIriSubjectTriples; the worker now TRANSPORTS verifiedSelection.consumedUnpersistedMetaTriples on the main path (no recompute), and its early-return branches (empty page / data-without-meta) still set 0 explicitly (tsc-enforced at every branch). The per-reason counts stay as diagnostics. Value-identical: same sum, computed where the drops are classified. Scope: only the count's OWNERSHIP moves. The larger prepareDurableMeta intake centralization (filter/drop/count of the verification-flow) stays deferred in #1943. Tests: added a durable-integrity assertion that selection.consumedUnpersistedMetaTriples === droppedSyncControlTriples + droppedNonIriSubjectTriples across pure-control / all-non-IRI / mixed pages; mutation-proven (drop the +nonIri addend → the all-non-IRI + mixed cases fail). Existing worker-aggregate + requester cursor-advance tests stay green (behavior unchanged). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agent/src/sync-verify-worker-impl.ts | 6 ++- packages/agent/src/sync/durable-integrity.ts | 20 ++++++++- .../sync-control-metadata-admission.test.ts | 41 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/packages/agent/src/sync-verify-worker-impl.ts b/packages/agent/src/sync-verify-worker-impl.ts index e007ce0222..ab28024a7b 100644 --- a/packages/agent/src/sync-verify-worker-impl.ts +++ b/packages/agent/src/sync-verify-worker-impl.ts @@ -389,8 +389,10 @@ function processDurableBatch( verifiedGraphScopedDataGraphs: verifiedSelection.verifiedGraphScopedDataGraphs, droppedSyncControlTriples: verifiedSelection.droppedSyncControlTriples, droppedNonIriSubjectTriples: verifiedSelection.droppedNonIriSubjectTriples, - consumedUnpersistedMetaTriples: - verifiedSelection.droppedSyncControlTriples + verifiedSelection.droppedNonIriSubjectTriples, + // Transport the verifier-owned aggregate (#1921) — do NOT recompute the sum + // here. The early-return branches above (empty page / data-without-meta) + // bypass selection and set consumedUnpersistedMetaTriples: 0 explicitly. + consumedUnpersistedMetaTriples: verifiedSelection.consumedUnpersistedMetaTriples, verifiedPrivateOnlyResponses: verifiedFullyPrivateResponse ? 1 : 0, totalFetchedDataQuads, totalFetchedMetaQuads, diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index a607dc2337..624ed7c215 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -102,10 +102,19 @@ export interface DurableIntegritySelection { dataIndexes: number[]; metaIndexes: number[]; rejected: number; - /** Unauthenticated cursor/routing rows deliberately consumed but not persisted. */ + /** Unauthenticated cursor/routing rows deliberately consumed but not persisted (diagnostic). */ droppedSyncControlTriples: number; - /** Non-IRI (blank-node/literal) `_meta` subject rows dropped at peer ingest (#1921). */ + /** Non-IRI (blank-node/literal) `_meta` subject rows dropped at peer ingest (#1921) (diagnostic). */ droppedNonIriSubjectTriples: number; + /** + * Reason-agnostic aggregate of `_meta` rows deliberately consumed but NOT + * persisted (= droppedSyncControlTriples + droppedNonIriSubjectTriples). Owned + * here by the verifier that classifies the drops (#1921): the worker transports + * it and the requester uses it as the single meta-checkpoint-advance signal, so + * checkpoint policy never has to enumerate verifier discard reasons. The two + * per-reason fields above stay as diagnostics. + */ + consumedUnpersistedMetaTriples: number; /** Verified V2 assets whose exact public assertion graph is intentionally empty. */ verifiedZeroPublicAssets: number; /** Exact assertion graphs whose V2 descriptors and fetched payload verified. */ @@ -374,6 +383,7 @@ export function selectVerifiedDurableSyncQuads( rejected: 1, droppedSyncControlTriples: 0, droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -385,6 +395,7 @@ export function selectVerifiedDurableSyncQuads( rejected: 0, droppedSyncControlTriples: 0, droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -414,6 +425,7 @@ export function selectVerifiedDurableSyncQuads( rejected: 1, droppedSyncControlTriples: 0, droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -434,6 +446,7 @@ export function selectVerifiedDurableSyncQuads( rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: selectedMetadata.droppedControls + selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -1059,6 +1072,7 @@ function selectVerifiedQuads( rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: selectedMetadata.droppedControls + selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1071,6 +1085,7 @@ function selectVerifiedQuads( rejected, droppedSyncControlTriples: 0, droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, // Keep this list aligned with the selected data + metadata indexes. // A fatal batch deliberately selects neither. Returning the names of @@ -1098,6 +1113,7 @@ function selectVerifiedQuads( rejected, droppedSyncControlTriples: selectedMetadata.droppedControls, droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: selectedMetadata.droppedControls + selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, diff --git a/packages/agent/test/sync-control-metadata-admission.test.ts b/packages/agent/test/sync-control-metadata-admission.test.ts index 50d857ca3b..fc765e50dc 100644 --- a/packages/agent/test/sync-control-metadata-admission.test.ts +++ b/packages/agent/test/sync-control-metadata-admission.test.ts @@ -460,4 +460,45 @@ describe('durable sync control metadata admission', () => { expect(selection.droppedNonIriSubjectTriples).toBe(2); expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); }); + + it('owns consumedUnpersistedMetaTriples as the sum of the per-reason drop counts (#1921)', () => { + // The verifier owns the checkpoint aggregate: it must always equal + // droppedSyncControlTriples + droppedNonIriSubjectTriples across every drop + // composition — pure sync-control, all-non-IRI, and mixed. + const controlSubject = 'urn:system:unverified-control'; + + // (a) pure sync-control: 3 unauthenticated controls dropped, 0 non-IRI. + const controlOnly = selectVerifiedDurableSyncQuads([], [ + quad(controlSubject, `${DKG}status`, '"keep"'), + quad(controlSubject, `${DKG}batchId`, integer(999n)), + quad(controlSubject, `${DKG}assertionGraph`, 'urn:attacker:graph'), + quad(controlSubject, `${DKG}assertionVersion`, integer(999n)), + ], true); + expect(controlOnly.droppedSyncControlTriples).toBe(3); + expect(controlOnly.droppedNonIriSubjectTriples).toBe(0); + expect(controlOnly.consumedUnpersistedMetaTriples) + .toBe(controlOnly.droppedSyncControlTriples + controlOnly.droppedNonIriSubjectTriples); + + // (b) all-non-IRI: 0 controls, 2 non-IRI dropped. + const nonIriOnly = selectVerifiedDurableSyncQuads([], [ + quad('_:injected', `${DKG}status`, '"drop"'), + quad('"literal-subject"', `${DKG}status`, '"drop-too"'), + ], true); + expect(nonIriOnly.droppedSyncControlTriples).toBe(0); + expect(nonIriOnly.droppedNonIriSubjectTriples).toBe(2); + expect(nonIriOnly.consumedUnpersistedMetaTriples) + .toBe(nonIriOnly.droppedSyncControlTriples + nonIriOnly.droppedNonIriSubjectTriples); + + // (c) mixed: both reasons contribute. + const mixed = selectVerifiedDurableSyncQuads([], [ + quad(controlSubject, `${DKG}batchId`, integer(999n)), + quad(controlSubject, `${DKG}assertionGraph`, 'urn:attacker:graph'), + quad('_:injected', `${DKG}status`, '"drop"'), + quad('"literal-subject"', `${DKG}label`, '"drop-too"'), + ], true); + expect(mixed.droppedSyncControlTriples).toBeGreaterThan(0); + expect(mixed.droppedNonIriSubjectTriples).toBe(2); + expect(mixed.consumedUnpersistedMetaTriples) + .toBe(mixed.droppedSyncControlTriples + mixed.droppedNonIriSubjectTriples); + }); });