diff --git a/packages/agent/src/sync-verify-worker-impl.ts b/packages/agent/src/sync-verify-worker-impl.ts index 6f42cdf6fc..ab28024a7b 100644 --- a/packages/agent/src/sync-verify-worker-impl.ts +++ b/packages/agent/src/sync-verify-worker-impl.ts @@ -307,6 +307,8 @@ function processDurableBatch( verifiedMetaIndexes: [], verifiedGraphScopedDataGraphs: [], droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedPrivateOnlyResponses: 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -330,6 +332,8 @@ function processDurableBatch( verifiedMetaIndexes: [], verifiedGraphScopedDataGraphs: [], droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedPrivateOnlyResponses: 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -384,6 +388,11 @@ function processDurableBatch( verifiedMetaIndexes: verifiedSelection.metaIndexes, verifiedGraphScopedDataGraphs: verifiedSelection.verifiedGraphScopedDataGraphs, droppedSyncControlTriples: verifiedSelection.droppedSyncControlTriples, + droppedNonIriSubjectTriples: verifiedSelection.droppedNonIriSubjectTriples, + // Transport the verifier-owned aggregate (#1921) — do NOT recompute the sum + // here. The early-return branches above (empty page / data-without-meta) + // bypass selection and set consumedUnpersistedMetaTriples: 0 explicitly. + consumedUnpersistedMetaTriples: verifiedSelection.consumedUnpersistedMetaTriples, verifiedPrivateOnlyResponses: verifiedFullyPrivateResponse ? 1 : 0, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -423,6 +432,8 @@ export function processDurableBatchForWire( verifiedMetaIndexes, verifiedGraphScopedDataGraphs, droppedSyncControlTriples, + droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples, verifiedPrivateOnlyResponses, totalFetchedDataQuads, totalFetchedMetaQuads, @@ -439,6 +450,8 @@ export function processDurableBatchForWire( verifiedMetaIndexes, verifiedGraphScopedDataGraphs, droppedSyncControlTriples, + droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples, verifiedPrivateOnlyResponses, totalFetchedDataQuads, totalFetchedMetaQuads, diff --git a/packages/agent/src/sync-verify-worker.ts b/packages/agent/src/sync-verify-worker.ts index 83107e99bf..cf8e7ff705 100644 --- a/packages/agent/src/sync-verify-worker.ts +++ b/packages/agent/src/sync-verify-worker.ts @@ -46,8 +46,23 @@ export interface DurableBatchProcessResult { verifiedData: Quad[]; verifiedMeta: Quad[]; verifiedGraphScopedDataGraphs: string[]; - /** Metadata controls deliberately consumed after failing authentication. */ + /** Metadata controls deliberately consumed after failing authentication (diagnostic). */ droppedSyncControlTriples: number; + /** + * Non-IRI (blank-node/literal) `_meta` subject rows deliberately dropped at + * ingest (#1921) — a verifier-side diagnostic count. + */ + droppedNonIriSubjectTriples: number; + /** + * Reason-agnostic aggregate of meta rows the verifier deliberately CONSUMED + * but did not persist (unverified sync controls + non-IRI subjects). This is + * the single count the requester uses to decide whether a fully-discarded + * metadata-only page still advances the meta checkpoint (rather than pinning + * durable sync on the same page). Keeping the per-reason counts above as + * diagnostics only keeps checkpoint orchestration decoupled from verifier + * discard policy. + */ + consumedUnpersistedMetaTriples: number; /** Clean batches containing verified V2 assets with no public assertion triples. */ verifiedPrivateOnlyResponses: number; totalFetchedDataQuads: number; diff --git a/packages/agent/src/sync/durable-integrity.ts b/packages/agent/src/sync/durable-integrity.ts index ab6ee9f2f7..624ed7c215 100644 --- a/packages/agent/src/sync/durable-integrity.ts +++ b/packages/agent/src/sync/durable-integrity.ts @@ -102,8 +102,19 @@ export interface DurableIntegritySelection { dataIndexes: number[]; metaIndexes: number[]; rejected: number; - /** Unauthenticated cursor/routing rows deliberately consumed but not persisted. */ + /** Unauthenticated cursor/routing rows deliberately consumed but not persisted (diagnostic). */ droppedSyncControlTriples: number; + /** Non-IRI (blank-node/literal) `_meta` subject rows dropped at peer ingest (#1921) (diagnostic). */ + droppedNonIriSubjectTriples: number; + /** + * Reason-agnostic aggregate of `_meta` rows deliberately consumed but NOT + * persisted (= droppedSyncControlTriples + droppedNonIriSubjectTriples). Owned + * here by the verifier that classifies the drops (#1921): the worker transports + * it and the requester uses it as the single meta-checkpoint-advance signal, so + * checkpoint policy never has to enumerate verifier discard reasons. The two + * per-reason fields above stay as diagnostics. + */ + consumedUnpersistedMetaTriples: number; /** Verified V2 assets whose exact public assertion graph is intentionally empty. */ verifiedZeroPublicAssets: number; /** Exact assertion graphs whose V2 descriptors and fetched payload verified. */ @@ -171,7 +182,13 @@ export function planBoundedGraphScopedDurableBatch( || metaQuads.length === 0 ) return null; - const metadata = indexIntegrityMetadata(dataQuads, metaQuads); + // #1921 — verification must never see non-IRI `_meta` subjects: a peer's + // `_:bad dkg:partOf ""` row would otherwise be scanned by + // readIntegrityMetadata and falsely invalidate the valid graph-scoped UAL. + // Sanitize before indexing. The bounded planner returns only data offsets, + // so dropping non-IRI meta rows here is loss-free. + const iriMetaQuads = metaQuads.filter((quad) => isIriMetaSubject(quad.subject)); + const metadata = indexIntegrityMetadata(dataQuads, iriMetaQuads); const parsed = readIntegrityMetadata(metadata, false); if ( parsed.fatalUnscopedFailure @@ -365,6 +382,8 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 1, droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -375,13 +394,25 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 0, droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, }; } - const metadata = indexIntegrityMetadata(dataQuads, metaQuads); + // #1921 — sanitize the verification inputs ONCE at this boundary so a non-IRI + // `_meta` subject can neither authenticate data (it never becomes a candidate) + // NOR poison verification (the readIntegrityMetadata PART_OF scan and + // verifyLegacyCandidates' raw scan never see it, so a `_:bad dkg:partOf + // ""` row cannot falsely invalidate a valid batch). Admission below + // deliberately runs on the ORIGINAL metaQuads: the selectors still drop+count + // non-IRI rows (persist-drop + meta-cursor advance) and index into the + // original array. The verification outcome is subject-keyed, so no positional + // remap is needed across the sanitized/original split. + const iriMetaQuads = metaQuads.filter((quad) => isIriMetaSubject(quad.subject)); + const metadata = indexIntegrityMetadata(dataQuads, iriMetaQuads); if (metadata.merkleSubjects.size === 0 && metadata.markerSubjects.size === 0) { if (!acceptUnverified && dataQuads.length > 0) { logs.push({ @@ -393,6 +424,8 @@ export function selectVerifiedDurableSyncQuads( metaIndexes: [], rejected: 1, droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -406,11 +439,14 @@ export function selectVerifiedDurableSyncQuads( new Set(), ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjectTriples); return { dataIndexes: allIndexes(dataQuads), metaIndexes: selectedMetadata.indexes, rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, + droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: selectedMetadata.droppedControls + selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: 0, verifiedGraphScopedDataGraphs: [], logs, @@ -427,7 +463,7 @@ export function selectVerifiedDurableSyncQuads( ); const legacy = verifyLegacyCandidates( dataQuads, - metaQuads, + iriMetaQuads, metadata, parsed.candidates, acceptUnverified, @@ -483,6 +519,16 @@ function indexIntegrityMetadata( const merkleSubjects = new Set(); const markerSubjects = new Set(); + // #1921 PRECONDITION: callers MUST pass IRI-sanitized `_meta` quads. Both + // current callers do — selectVerifiedDurableSyncQuads and + // planBoundedGraphScopedDurableBatch filter non-IRI subjects at their boundary + // before indexing — and any NEW caller MUST too. This function no longer + // filters internally, so a non-IRI subject reaching here would re-enter + // candidacy AND the metaBySubject-based verification scans + // (readIntegrityMetadata's PART_OF scan, parseGraphScopedDescriptor), letting a + // peer's blank-node/literal subject authenticate data OR poison a valid batch. + // Admission (the selectors) deliberately runs on the ORIGINAL metaQuads and is + // where non-IRI rows are dropped + counted. for (const quad of metaQuads) { if (quad.predicate === MERKLE_ROOT) { merkleSubjects.add(quad.subject); @@ -1019,11 +1065,14 @@ function selectVerifiedQuads( outcome.kaToKc, ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjectTriples); return { dataIndexes: allIndexes(dataQuads), metaIndexes: selectedMetadata.indexes, rejected: 0, droppedSyncControlTriples: selectedMetadata.droppedControls, + droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: selectedMetadata.droppedControls + selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1035,6 +1084,8 @@ function selectVerifiedQuads( metaIndexes: [], rejected, droppedSyncControlTriples: 0, + droppedNonIriSubjectTriples: 0, + consumedUnpersistedMetaTriples: 0, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, // Keep this list aligned with the selected data + metadata indexes. // A fatal batch deliberately selects neither. Returning the names of @@ -1054,12 +1105,15 @@ function selectVerifiedQuads( outcome.authenticatedMetadataUals, ); logDroppedSyncControls(logs, selectedMetadata.droppedControls); + logDroppedNonIriMetaSubjects(logs, selectedMetadata.droppedNonIriSubjectTriples); return { dataIndexes, metaIndexes: selectedMetadata.indexes, rejected, droppedSyncControlTriples: selectedMetadata.droppedControls, + droppedNonIriSubjectTriples: selectedMetadata.droppedNonIriSubjectTriples, + consumedUnpersistedMetaTriples: selectedMetadata.droppedControls + selectedMetadata.droppedNonIriSubjectTriples, verifiedZeroPublicAssets: outcome.verifiedZeroPublicAssets, verifiedGraphScopedDataGraphs, logs, @@ -1072,11 +1126,23 @@ function selectAdmittedMetadataIndexes( admittedMetadataUals: ReadonlySet, kaToKc: ReadonlyMap, authenticatedMetadataUals: ReadonlySet, -): { indexes: number[]; droppedControls: number } { +): { indexes: number[]; droppedControls: number; droppedNonIriSubjectTriples: number } { const indexes: number[] = []; let droppedControls = 0; + let droppedNonIriSubjectTriples = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; + // #1921 — admission runs on the ORIGINAL metaQuads (verification already ran + // on the IRI-sanitized set at the boundary). A non-IRI descriptive row would + // otherwise reach the descriptive fall-through below and be persisted, so + // drop + count it here — that keeps it out of the store AND feeds the + // meta-cursor consumed-row count (checkpoint advance). It could not reach the + // merkle/marker branch regardless: the boundary sanitize keeps non-IRI + // subjects out of `merkleSubjects`/`markerSubjects`. + if (!isIriMetaSubject(quad.subject)) { + droppedNonIriSubjectTriples += 1; + continue; + } if ( metadata.merkleSubjects.has(quad.subject) || metadata.markerSubjects.has(quad.subject) @@ -1130,7 +1196,7 @@ function selectAdmittedMetadataIndexes( } indexes.push(index); } - return { indexes, droppedControls }; + return { indexes, droppedControls, droppedNonIriSubjectTriples }; } function selectSystemOverrideMetadataIndexes( @@ -1138,11 +1204,20 @@ function selectSystemOverrideMetadataIndexes( metadata: IntegrityMetadataIndex, authenticatedMetadataUals: ReadonlySet, kaToKc: ReadonlyMap, -): { indexes: number[]; droppedControls: number } { +): { indexes: number[]; droppedControls: number; droppedNonIriSubjectTriples: number } { const indexes: number[] = []; let droppedControls = 0; + let droppedNonIriSubjectTriples = 0; for (let index = 0; index < metaQuads.length; index++) { const quad = metaQuads[index]!; + // #1921 — reject a non-IRI durable `_meta` subject at ingest. This terminal + // system-CG selector admits every non-control row, so without this guard a + // blank-node subject bearing ANY descriptive or integrity predicate + // (e.g. `dkg:merkleRoot`) would be persisted and later served back. + if (!isIriMetaSubject(quad.subject)) { + droppedNonIriSubjectTriples += 1; + continue; + } if ( DURABLE_SYNC_CONTROL_PREDICATE_SET.has(quad.predicate) && !isGraphSealDescriptiveVersion(quad, metadata) @@ -1158,7 +1233,7 @@ function selectSystemOverrideMetadataIndexes( } indexes.push(index); } - return { indexes, droppedControls }; + return { indexes, droppedControls, droppedNonIriSubjectTriples }; } /** @@ -1247,6 +1322,17 @@ function logDroppedSyncControls( }); } +function logDroppedNonIriMetaSubjects( + logs: DurableIntegrityLogEntry[], + dropped: number, +): void { + if (dropped === 0) return; + logs.push({ + level: 'warn', + message: `Dropped ${dropped} non-IRI durable _meta subject triple(s) from peer ingest (#1921)`, + }); +} + function isDetachedLegacyProjectionGraph(graph: string): boolean { if (!graph.startsWith('did:dkg:context-graph:')) return false; return graph.endsWith('/_catalog') @@ -1394,6 +1480,25 @@ function stripLiteral(raw: string): string { return match ? match[1]! : raw; } +/** + * A durable `_meta` subject must be an IRI. Conforming writers only emit IRI + * subjects (metadata generators build deterministic UALs; the publisher rejects + * blank nodes; SWM writers skolemize before storage). A blank-node (`_:…`) or + * literal (`"…"`) subject is only reachable via unverified peer-ingest and has + * no trustworthy, stable identity, so it is dropped at ingest (#1921) rather + * than persisted. Mirrors the responder's `isIriTerm` (graph-plan.ts) so ingest + * and read agree on the contract. + */ +function isIriMetaSubject(term: string): boolean { + // Defensive on `term`: this runs at the verification-input boundary filters + // (selectVerifiedDurableSyncQuads and planBoundedGraphScopedDurableBatch, both + // on RAW fetched meta) and in the admission selectors. A conforming quad always + // carries a non-empty string subject; tolerate a malformed one (treat as + // non-IRI → drop) instead of throwing. + return typeof term === 'string' && term.length > 0 + && !term.startsWith('_:') && !term.startsWith('"'); +} + function findLegacyRootOwner(subject: string, roots: ReadonlySet): string | undefined { for (const root of roots) { if (subject === root || subject.startsWith(`${root}${SKOLEM_SUFFIX}`)) return root; diff --git a/packages/agent/src/sync/requester/durable-sync.ts b/packages/agent/src/sync/requester/durable-sync.ts index a710dcfdca..9705ac3aeb 100644 --- a/packages/agent/src/sync/requester/durable-sync.ts +++ b/packages/agent/src/sync/requester/durable-sync.ts @@ -117,7 +117,13 @@ interface DurableSyncContext { verifiedData: Quad[]; verifiedMeta: Quad[]; verifiedGraphScopedDataGraphs?: string[]; - droppedSyncControlTriples?: number; + /** + * Worker-owned aggregate of meta rows deliberately consumed but not + * persisted. REQUIRED (#1921): it is the single checkpoint-advance signal, + * so every producer must set it — an optional field silently reading 0 + * would let the meta cursor pin with no type error. + */ + consumedUnpersistedMetaTriples: number; totalFetchedDataQuads: number; totalFetchedMetaQuads: number; rejectedKcs: number; @@ -472,12 +478,20 @@ export async function runDurableSync( }; const metadataOnlyResponse = processed.metaOnlyResponses > 0; - const droppedSyncControlTriples = processed.droppedSyncControlTriples ?? 0; + // The worker reports, as ONE reason-agnostic count, how many fetched meta + // rows the verifier deliberately consumed but did NOT persist — unverified + // sync controls plus non-IRI `_meta` subjects (#1921). A metadata-only page + // discarded ENTIRELY this way carries no verifiedMeta, so the meta cursor + // must still advance or durable sync pins on the same page. Depending on the + // aggregate (not per-reason counters) keeps checkpoint orchestration + // decoupled from verifier discard policy; the per-reason counts remain as + // verifier-side diagnostics only. + const consumedUnpersistedMetaTriples = processed.consumedUnpersistedMetaTriples; const discardedOnlyMetadataResponse = metadataOnlyResponse && processed.verifiedData.length === 0 && processed.verifiedMeta.length === 0 - && droppedSyncControlTriples > 0 - && droppedSyncControlTriples === processed.totalFetchedMetaQuads; + && consumedUnpersistedMetaTriples > 0 + && consumedUnpersistedMetaTriples === processed.totalFetchedMetaQuads; const updateMetaCheckpoint = batchVerifiedCleanly && processed.dataRejectedMissingMeta === 0 && ( diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 03673ecc26..a6ad392d99 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -613,13 +613,16 @@ type MetaOversizedSubjectPolicy = 'byte-fit' | 'fail-loud'; * partial-metadata loss + a #1788 seal split). Failing loud is strictly better * than silent loss here: a byte-budget-negotiating requester never hits this (it * uses empty=EOF pagination), and the oversized `_meta` subject itself is only - * reachable via unverified peer-ingest, fixed at the root by #1921. + * reachable via unverified peer-ingest. Closing that SIZE vector at the root is + * tracked by #1923 (SyncPagePolicy) — it is orthogonal to #1921's IRI-only + * ingest guard, since a peer can inject a valid-IRI subject bearing a giant + * literal. * * Contract: this is a HARD, NON-RETRYABLE failure. The sync responder surfaces it * as `outcome:'error'` and re-throws it (it is deliberately NOT wrapped in a * retryable error, and NEVER converted to an empty — EOF — body), because a retry * cannot make an oversized subject servable to a legacy requester; the only - * resolutions are a requester upgrade or the #1921 ingest fix. The budget it is + * resolutions are a requester upgrade or the #1923 ingest fix. The budget it is * checked against is `SYNC_BYTE_BUDGET_RESPONSE_BYTES` (the router read cap minus * the frame headroom) — the largest response body guaranteed to fit one transport * frame — so a page within it is always sendable and only a genuinely oversized @@ -636,7 +639,7 @@ export class DurableMetaPageFrameError extends Error { + `non-negotiated (legacy) requester: a subject-atomic page of ${params.bytes} bytes ` + `exceeds the ${params.limit}-byte frame budget. A byte-budget-negotiating requester ` + `(testnet-canary+) paginates this via empty=EOF; upgrade the requester or reject the ` - + `oversized _meta subject at ingest (#1921).`, + + `oversized _meta subject at ingest (#1923).`, ); this.name = 'DurableMetaPageFrameError'; this.contextGraphId = params.contextGraphId; @@ -3795,10 +3798,13 @@ async function readDurableMetaRowsPage( * but within a single self-contained window its label — and therefore the * `(g, s)` boundary comparison — is consistent. Conforming writers only emit * IRI `_meta` subjects (metadata generators build deterministic IRIs; the - * publisher rejects blank nodes), but a blank-node subject is reachable via the - * unverified system-CG peer-ingest path, so subject atomicity must cover it too; - * enforcing the IRI invariant at ingest (defense-in-depth) is tracked - * separately. + * publisher rejects blank nodes). NEW unverified peer ingest now enforces this + * IRI invariant at the durable-meta selection (#1921), so it no longer admits a + * blank-node subject — but that guard operates on incoming quads only and does + * NOT sweep already-persisted data, so a blank-node `_meta` subject written by a + * pre-fix peer may still reside in the store. Subject atomicity must therefore + * still cover it, and this per-query relabel accommodation stays as + * defense-in-depth. * * `_meta` subjects are small (a seal is 14 quads ≈ a few KB), so this converges * in about one read. The returned page is BOTH subject-atomic AND ≤ diff --git a/packages/agent/test/durable-sync-since-threading.test.ts b/packages/agent/test/durable-sync-since-threading.test.ts index 95996b2288..30d25f6919 100644 --- a/packages/agent/test/durable-sync-since-threading.test.ts +++ b/packages/agent/test/durable-sync-since-threading.test.ts @@ -171,6 +171,7 @@ function makeContext(options: { return { verifiedData, verifiedMeta, + consumedUnpersistedMetaTriples: 0, totalFetchedDataQuads: options.processResult?.totalFetchedDataQuads ?? dataQuads.length, totalFetchedMetaQuads: options.processResult?.totalFetchedMetaQuads ?? metaQuads.length, rejectedKcs: 0, diff --git a/packages/agent/test/rootless-durable-bounded-progress.test.ts b/packages/agent/test/rootless-durable-bounded-progress.test.ts index c06dcfa711..9d2a9441af 100644 --- a/packages/agent/test/rootless-durable-bounded-progress.test.ts +++ b/packages/agent/test/rootless-durable-bounded-progress.test.ts @@ -148,6 +148,50 @@ describe('bounded rootless durable progress', () => { ]); }); + it('ignores a non-IRI dkg:partOf poison row and still projects the valid boundary (#1921)', () => { + // A peer can attach `_:bad dkg:partOf ""` to a valid graph-scoped + // manifest. Without the bounded planner's #1921 verification-input sanitize, + // readIntegrityMetadata's PART_OF scan would invalidate that valid UAL → + // planBoundedGraphScopedDurableBatch returns null → timed-out graph-scoped + // progress is lost (sync pins). The sanitize keeps non-IRI subjects out of + // verification, so the planner still projects the same safe prefix as the + // clean case above. Mutation check: remove the iriMetaQuads filter from + // planBoundedGraphScopedDurableBatch and this returns null. + const fixtures = orderedAssets(); + const meta = fixtures.flatMap((entry) => entry.meta); + const poison: Quad = { + subject: '_:bad', + predicate: 'http://dkg.io/ontology/partOf', + object: `"${fixtures[0]!.ual}"`, + graph: `${CONTEXT_GRAPH_URI}/_meta`, + }; + const rawData = [ + ...fixtures[0]!.payload, + ...fixtures[1]!.payload, + ...fixtures[2]!.payload.slice(0, 2), + ]; + + const plan = planBoundedGraphScopedDurableBatch( + rawData, + [...meta, poison], + 0, + rawData.length, + false, + ); + + expect(plan).not.toBeNull(); + expect(plan?.safeNextOffset).toBe(8); + expect(plan?.completedGraphCount).toBe(2); + expect(plan?.changedDataGraphs).toEqual([ + fixtures[0]!.graph, + fixtures[1]!.graph, + ]); + expect(plan?.dataQuads).toEqual([ + ...fixtures[0]!.payload, + ...fixtures[1]!.payload, + ]); + }); + it('replays one complete graph when timeout lands exactly on a boundary', () => { const fixtures = orderedAssets(); const meta = fixtures.flatMap((entry) => entry.meta); diff --git a/packages/agent/test/sync-control-metadata-admission.test.ts b/packages/agent/test/sync-control-metadata-admission.test.ts index 76d5ea18c7..fc765e50dc 100644 --- a/packages/agent/test/sync-control-metadata-admission.test.ts +++ b/packages/agent/test/sync-control-metadata-admission.test.ts @@ -259,4 +259,246 @@ describe('durable sync control metadata admission', () => { expect(selectedMeta(data, meta)).toEqual(meta); }); + + // #1921 — a durable `_meta` subject must be a conforming IRI. A peer can only + // reach the unverified system-CG ingest path with a blank-node (or literal) + // subject; it has no trustworthy identity, so it is dropped (never persisted, + // never skolemized) at the selection chokepoint. The guard sits at the top of + // BOTH selector loops, so it fires on every guarded call site. + const IRI_SUBJECT = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/50'; + + it('drops a forged-integrity blank-node merkleRoot subject without persisting it (#1921)', () => { + // A blank node bearing an INTEGRITY predicate. The #1921 candidate-gate in + // `indexIntegrityMetadata` excludes it from `merkleSubjects`, so it never + // becomes a verification candidate; with no verified descriptor the batch + // takes the descriptor-less admission path, where the selector guard drops + // the blank-node rows (persist-drop + count) and keeps the conforming IRI + // row. Contrast the IRI-subject case "does not let the system-graph override + // authenticate malformed legacy controls" (which KEEPS the merkleRoot). + const injected = '_:injected'; + const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"legit"'); + const meta = [ + quad(injected, `${DKG}merkleRoot`, `"${'00'.repeat(32)}"`), + quad(injected, `${DKG}status`, '"forged"'), + quad(injected, `${DKG}batchId`, integer(999n)), + keptRow, + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); + expect(selection.droppedNonIriSubjectTriples).toBe(3); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + for (const [label, injected] of [ + ['blank-node', '_:injected'], + ['literal', '"forged-subject"'], + ] as const) { + it(`does not let a ${label} _meta subject with dkg:merkleRoot authenticate its data (#1921)`, () => { + // #1921 root fix (candidate-gate): a non-IRI subject bearing dkg:merkleRoot + // must never become a verification candidate. Otherwise it self-consistently + // authenticates its bound DATA (the claimed root is peer-supplied, not + // on-chain-anchored), the data is admitted, and only the METADATA is later + // dropped — persisting orphaned, peer-forged data. With the gate the batch + // fails closed: no verified descriptor, so the data is rejected and NOT + // selected. Distinct from the persist-drop/cursor tests above. + const root = 'urn:legacy:root'; + const data = [quad(root, 'urn:example:value', '"legacy"', CONTEXT_GRAPH)]; + const merkleRoot = `"${toHex(computeFlatKCRootV10(data, []))}"`; + const meta = [ + quad(injected, `${DKG}merkleRoot`, merkleRoot), + quad(injected, `${DKG}rootEntity`, root), + quad(injected, `${DKG}batchId`, integer(41n)), + ]; + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + // Fail closed: the non-IRI envelope authenticates nothing, so neither the + // data nor its metadata is persisted. + expect(selection.rejected).toBe(1); + expect(selection.dataIndexes).toEqual([]); + expect(selection.metaIndexes).toEqual([]); + }); + } + + it('drops a non-IRI _meta subject on a descriptor-less system page (#1921)', () => { + // The :401 no-merkle/no-marker branch: `selectAdmittedMetadataIndexes` runs + // with empty admission sets, so every row falls through the descriptive + // path. Exercises the third guarded call site's warn wiring. + const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"keep"'); + const meta = [ + keptRow, + quad('_:injected', `${DKG}status`, '"drop"'), + quad('_:injected', `${DKG}label`, '"drop-too"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); + expect(selection.droppedNonIriSubjectTriples).toBe(2); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('counts an all-non-IRI system-CG metadata-only page as fully consumed so the cursor advances (#1921)', () => { + // Livelock-fix-intact guard: after the candidate-gate, an acceptUnverified + // (system-CG) metadata-only page consisting ENTIRELY of non-IRI subjects — + // even a forged dkg:merkleRoot — is neither rejected nor pinned. The forged + // merkle subject is excluded from candidacy, so no verified descriptor + // exists; with no data the batch is not rejected, and every row is dropped + // and COUNTED (droppedNonIriSubjectTriples === total). That equality is what + // lets the requester advance the meta cursor instead of re-fetching forever. + const meta = [ + quad('_:injected', `${DKG}merkleRoot`, `"${'00'.repeat(32)}"`), + quad('_:injected', `${DKG}status`, '"forged"'), + quad('"literal-subject"', `${DKG}status`, '"drop-too"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.rejected).toBe(0); + expect(selection.metaIndexes).toEqual([]); + expect(selection.droppedNonIriSubjectTriples).toBe(meta.length); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('keeps a conforming IRI _meta subject untouched (no #1921 drop)', () => { + const scope = createGraphKnowledgeAssetScope(UAL, '1'); + const assertionGraph = knowledgeAssetLayerGraphUri( + CONTEXT_GRAPH_ID, + MemoryLayer.VerifiableMemory, + scope, + ); + const data = [quad('urn:verified:entity', 'urn:example:value', '"verified"', assertionGraph)]; + const meta = generateGraphKnowledgeAssetMetadata({ + ual: UAL, + contextGraphId: CONTEXT_GRAPH_ID, + merkleRoot: computeFlatKCRootV10(data, []), + publisherPeerId: 'verified-publisher', + accessPolicy: 'public', + timestamp: new Date(0), + assertionVersion: '1', + publicTripleCount: data.length, + assertionGraph, + }, { status: 'tentative' }); + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + expect(selection.droppedNonIriSubjectTriples).toBe(0); + expect(selection.logs.every((entry) => !/non-IRI/.test(entry.message))).toBe(true); + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual(meta); + }); + + it('drops a literal `_meta` subject at ingest (#1921)', () => { + // The guard rejects BOTH blank-node and literal subjects as non-IRI. A + // literal subject term (begins with `"`) is malformed for a `_meta` subject + // and must be dropped, counted, and warned exactly like a blank node — this + // proves the second half of the advertised ingest contract. + const keptRow = quad(IRI_SUBJECT, `${DKG}status`, '"keep"'); + const meta = [ + keptRow, + quad('"literal-subject"', `${DKG}status`, '"drop"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptRow]); + expect(selection.droppedNonIriSubjectTriples).toBe(1); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('verifies a graph-scoped asset even when a non-IRI dkg:partOf row is present (#1921)', () => { + // A peer can attach `_:bad dkg:partOf ""` beside a valid + // graph-scoped descriptor. Before the #1921 verification-input sanitize, + // readIntegrityMetadata's PART_OF scan over metaBySubject saw that non-IRI + // row and falsely invalidated the valid UAL → the whole batch was rejected + // → durable sync pinned (a poison/DoS vector). The boundary sanitize keeps + // non-IRI subjects out of verification, so the asset still verifies while + // the bad row is dropped + counted (never persisted). + const scope = createGraphKnowledgeAssetScope(UAL, '1'); + const assertionGraph = knowledgeAssetLayerGraphUri(CONTEXT_GRAPH_ID, MemoryLayer.VerifiableMemory, scope); + const data = [quad('urn:verified:entity', 'urn:example:value', '"verified"', assertionGraph)]; + const verified = generateGraphKnowledgeAssetMetadata({ + ual: UAL, + contextGraphId: CONTEXT_GRAPH_ID, + merkleRoot: computeFlatKCRootV10(data, []), + publisherPeerId: 'verified-publisher', + accessPolicy: 'public', + timestamp: new Date(0), + assertionVersion: '1', + publicTripleCount: data.length, + assertionGraph, + }, { status: 'tentative' }); + const poison = quad('_:bad', `${DKG}partOf`, `"${UAL}"`); + const meta = [...verified, poison]; + + const selection = selectVerifiedDurableSyncQuads(data, meta, false); + + expect(selection.rejected).toBe(0); + expect(selection.dataIndexes).toEqual([0]); + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual(verified); + expect(selection.droppedNonIriSubjectTriples).toBe(1); + }); + + it('drops a non-IRI _meta subject on the system-override path (#1921)', () => { + // Directly exercises selectSystemOverrideMetadataIndexes: an IRI merkle + // subject that fails legacy verification (no rootEntity) forces the + // accept-unverified system-override path (rejected>0 && acceptUnverified), + // where every non-control row is otherwise admitted. The non-IRI rows must + // be dropped + counted on THIS branch, not persisted. + const iriSubject = 'urn:system:malformed-legacy'; + const keptMerkle = quad(iriSubject, `${DKG}merkleRoot`, `"${'00'.repeat(32)}"`); + const meta = [ + keptMerkle, + quad('_:injected', `${DKG}status`, '"drop"'), + quad('"literal-subject"', `${DKG}label`, '"drop-too"'), + ]; + + const selection = selectVerifiedDurableSyncQuads([], meta, true); + + expect(selection.metaIndexes.map((index) => meta[index]!)).toEqual([keptMerkle]); + expect(selection.droppedNonIriSubjectTriples).toBe(2); + expect(selection.logs.some((entry) => /non-IRI durable _meta subject/.test(entry.message))).toBe(true); + }); + + it('owns consumedUnpersistedMetaTriples as the sum of the per-reason drop counts (#1921)', () => { + // The verifier owns the checkpoint aggregate: it must always equal + // droppedSyncControlTriples + droppedNonIriSubjectTriples across every drop + // composition — pure sync-control, all-non-IRI, and mixed. + const controlSubject = 'urn:system:unverified-control'; + + // (a) pure sync-control: 3 unauthenticated controls dropped, 0 non-IRI. + const controlOnly = selectVerifiedDurableSyncQuads([], [ + quad(controlSubject, `${DKG}status`, '"keep"'), + quad(controlSubject, `${DKG}batchId`, integer(999n)), + quad(controlSubject, `${DKG}assertionGraph`, 'urn:attacker:graph'), + quad(controlSubject, `${DKG}assertionVersion`, integer(999n)), + ], true); + expect(controlOnly.droppedSyncControlTriples).toBe(3); + expect(controlOnly.droppedNonIriSubjectTriples).toBe(0); + expect(controlOnly.consumedUnpersistedMetaTriples) + .toBe(controlOnly.droppedSyncControlTriples + controlOnly.droppedNonIriSubjectTriples); + + // (b) all-non-IRI: 0 controls, 2 non-IRI dropped. + const nonIriOnly = selectVerifiedDurableSyncQuads([], [ + quad('_:injected', `${DKG}status`, '"drop"'), + quad('"literal-subject"', `${DKG}status`, '"drop-too"'), + ], true); + expect(nonIriOnly.droppedSyncControlTriples).toBe(0); + expect(nonIriOnly.droppedNonIriSubjectTriples).toBe(2); + expect(nonIriOnly.consumedUnpersistedMetaTriples) + .toBe(nonIriOnly.droppedSyncControlTriples + nonIriOnly.droppedNonIriSubjectTriples); + + // (c) mixed: both reasons contribute. + const mixed = selectVerifiedDurableSyncQuads([], [ + quad(controlSubject, `${DKG}batchId`, integer(999n)), + quad(controlSubject, `${DKG}assertionGraph`, 'urn:attacker:graph'), + quad('_:injected', `${DKG}status`, '"drop"'), + quad('"literal-subject"', `${DKG}label`, '"drop-too"'), + ], true); + expect(mixed.droppedSyncControlTriples).toBeGreaterThan(0); + expect(mixed.droppedNonIriSubjectTriples).toBe(2); + expect(mixed.consumedUnpersistedMetaTriples) + .toBe(mixed.droppedSyncControlTriples + mixed.droppedNonIriSubjectTriples); + }); }); diff --git a/packages/agent/test/sync-durable-worker-wire.test.ts b/packages/agent/test/sync-durable-worker-wire.test.ts index 31036e5781..59afbf7986 100644 --- a/packages/agent/test/sync-durable-worker-wire.test.ts +++ b/packages/agent/test/sync-durable-worker-wire.test.ts @@ -334,6 +334,68 @@ describe('durable sync worker result transport', () => { expect(wireResult.verifiedMetaIndexes).toEqual([]); expect(wireResult.droppedSyncControlTriples).toBe(meta.length); + // Pure-sync-control page: the requester-facing aggregate equals the + // sync-control diagnostic count, preserving the shipped checkpoint semantics. + expect(wireResult.consumedUnpersistedMetaTriples).toBe(meta.length); + expect(wireResult.metaOnlyResponses).toBe(1); + expect(wireResult.rejectedKcs).toBe(0); + }); + + it('reports discarded-only non-IRI meta subjects across the worker wire (#1921)', () => { + // Exercises the REAL selection -> processDurableBatchForWire path so a + // regression that drops droppedNonIriSubjectTriples between the selector and + // the wire result is caught (the requester-progress unit tests inject the + // count directly and cannot see this). A metadata-only page of purely + // non-IRI subjects must report the full dropped count so the requester can + // advance the meta cursor instead of pinning. + const meta: Quad[] = [ + { + subject: '_:injected', + predicate: 'http://dkg.io/ontology/status', + object: '"drop"', + graph: META_GRAPH, + }, + { + subject: '"literal-subject"', + predicate: 'http://dkg.io/ontology/status', + object: '"drop-too"', + graph: META_GRAPH, + }, + ]; + + const wireResult = processDurableBatchForWire([], meta, false); + + expect(wireResult.verifiedMetaIndexes).toEqual([]); + expect(wireResult.droppedNonIriSubjectTriples).toBe(meta.length); + // All-non-IRI page: the aggregate equals the non-IRI diagnostic count. + expect(wireResult.consumedUnpersistedMetaTriples).toBe(meta.length); + expect(wireResult.metaOnlyResponses).toBe(1); + expect(wireResult.rejectedKcs).toBe(0); + }); + + it('aggregates mixed control + non-IRI drops into consumedUnpersistedMetaTriples (#1921)', () => { + // Worker-level guard for the consolidated checkpoint contract: the single + // requester-facing aggregate equals the SUM of the per-reason diagnostic + // counts for a page mixing unverified sync controls and non-IRI subjects. + // Neutralizing the worker sum breaks this; the per-reason counts remain as + // diagnostics. + const control = 'urn:orphaned:sync-control'; + const meta: Quad[] = [ + { subject: control, predicate: 'http://dkg.io/ontology/batchId', object: '"999"^^', graph: META_GRAPH }, + { subject: control, predicate: 'http://dkg.io/ontology/assertionGraph', object: 'urn:attacker:graph', graph: META_GRAPH }, + { subject: '_:injected', predicate: 'http://dkg.io/ontology/status', object: '"drop"', graph: META_GRAPH }, + { subject: '"literal-subject"', predicate: 'http://dkg.io/ontology/status', object: '"drop-too"', graph: META_GRAPH }, + ]; + + const wireResult = processDurableBatchForWire([], meta, false); + + expect(wireResult.verifiedMetaIndexes).toEqual([]); + expect(wireResult.droppedSyncControlTriples).toBe(2); + expect(wireResult.droppedNonIriSubjectTriples).toBe(2); + expect(wireResult.consumedUnpersistedMetaTriples).toBe( + wireResult.droppedSyncControlTriples + wireResult.droppedNonIriSubjectTriples, + ); + expect(wireResult.consumedUnpersistedMetaTriples).toBe(meta.length); expect(wireResult.metaOnlyResponses).toBe(1); expect(wireResult.rejectedKcs).toBe(0); }); diff --git a/packages/agent/test/sync-requester-progress.test.ts b/packages/agent/test/sync-requester-progress.test.ts index 5916c69df7..5054b4e4c1 100644 --- a/packages/agent/test/sync-requester-progress.test.ts +++ b/packages/agent/test/sync-requester-progress.test.ts @@ -57,7 +57,7 @@ function durableProcessResult() { return { verifiedData: [] as Quad[], verifiedMeta: [] as Quad[], - droppedSyncControlTriples: 0, + consumedUnpersistedMetaTriples: 0, totalFetchedDataQuads: 0, totalFetchedMetaQuads: 0, rejectedKcs: 0, @@ -515,7 +515,9 @@ describe('sync requester progress accounting', () => { ...durableProcessResult(), emptyResponses: 0, metaOnlyResponses: 1, - droppedSyncControlTriples: 3, + // Pure-sync-control page: worker aggregates 3 discarded controls into + // consumedUnpersistedMetaTriples. Regression guard for the shipped path. + consumedUnpersistedMetaTriples: 3, totalFetchedMetaQuads: 3, }), storeInsert, @@ -556,7 +558,7 @@ describe('sync requester progress accounting', () => { ...durableProcessResult(), emptyResponses: 0, metaOnlyResponses: 1, - droppedSyncControlTriples: 2, + consumedUnpersistedMetaTriples: 2, totalFetchedMetaQuads: 3, }), storeInsert: async () => {}, @@ -571,6 +573,96 @@ describe('sync requester progress accounting', () => { expect(setCheckpoint.calls).toEqual([]); }); + it('advances the meta cursor when the whole page is non-IRI subjects dropped at ingest (#1921)', async () => { + // All-non-IRI metadata-only page: the worker aggregates every dropped row + // into consumedUnpersistedMetaTriples === totalFetchedMetaQuads and no meta + // is persisted. The requester must still advance the meta cursor (the rows + // were deliberately consumed) or durable sync pins on the same poisoned page. + const setCheckpoint = recorder((_key: string, _offset: number) => {}); + const deleteCheckpoint = recorder((_key: string) => {}); + const storeInsert = recorder(async (_quads: Quad[]) => {}); + const fetchSyncPages = recorder(async ( + _ctx: OperationContext, + _peer: string, + contextGraphId: string, + _includeSharedMemory: boolean, + phase: 'data' | 'meta', + ) => phase === 'meta' + ? pageResult(contextGraphId, phase, { nextOffset: 3, completed: false }) + : pageResult(contextGraphId, phase)); + + const summary = await runDurableSync({ + ctx, + remotePeerId: 'peer-a', + contextGraphIds: ['discarded-non-iri'], + createContextGraphSyncDeadline: () => Date.now() + 60_000, + fetchSyncPages, + processDurableBatchInWorker: async () => ({ + ...durableProcessResult(), + emptyResponses: 0, + metaOnlyResponses: 1, + consumedUnpersistedMetaTriples: 3, + totalFetchedMetaQuads: 3, + }), + storeInsert, + deleteCheckpoint, + setCheckpoint, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(summary.metaOnlyResponses).toBe(1); + expect(summary.checkpointAdvances).toBe(0); + expect(storeInsert.calls).toEqual([]); + expect(deleteCheckpoint.calls).toEqual([]); + expect(setCheckpoint.calls).toEqual([['discarded-non-iri:meta', 3]]); + }); + + it('advances the meta cursor when a mixed page is fully discarded by controls + non-IRI drops (#1921)', async () => { + // A mixed all-discarded page (some unverified controls + some non-IRI rows): + // the worker sums both reasons into consumedUnpersistedMetaTriples === the + // fetched total, so the requester advances the cursor rather than pinning. + const setCheckpoint = recorder((_key: string, _offset: number) => {}); + const deleteCheckpoint = recorder((_key: string) => {}); + const storeInsert = recorder(async (_quads: Quad[]) => {}); + const fetchSyncPages = recorder(async ( + _ctx: OperationContext, + _peer: string, + contextGraphId: string, + _includeSharedMemory: boolean, + phase: 'data' | 'meta', + ) => phase === 'meta' + ? pageResult(contextGraphId, phase, { nextOffset: 3, completed: false }) + : pageResult(contextGraphId, phase)); + + const summary = await runDurableSync({ + ctx, + remotePeerId: 'peer-a', + contextGraphIds: ['discarded-mixed'], + createContextGraphSyncDeadline: () => Date.now() + 60_000, + fetchSyncPages, + processDurableBatchInWorker: async () => ({ + ...durableProcessResult(), + emptyResponses: 0, + metaOnlyResponses: 1, + consumedUnpersistedMetaTriples: 3, + totalFetchedMetaQuads: 3, + }), + storeInsert, + deleteCheckpoint, + setCheckpoint, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(summary.metaOnlyResponses).toBe(1); + expect(storeInsert.calls).toEqual([]); + expect(deleteCheckpoint.calls).toEqual([]); + expect(setCheckpoint.calls).toEqual([['discarded-mixed:meta', 3]]); + }); + it('deletes only the durable meta checkpoint after completing metadata-only responses', async () => { const metaQuad = quad('meta-only-complete-meta'); const storeInsert = recorder(async (_quads: Quad[]) => {});