From 0b42ebc7bb5cb78d627466bbec1c46857d4ce661 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 10:02:27 +0200 Subject: [PATCH 01/48] fix(agent): safely clean finalized receiver SWM --- packages/agent/src/dkg-agent-swm-substrate.ts | 2 + packages/agent/src/finalization-handler.ts | 281 +++++++++++++++++- .../ka-graph-finalization-handler.test.ts | 95 +++++- 3 files changed, 370 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 4893d23842..9646b1276c 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -1654,6 +1654,8 @@ export class SwmSubstrateMethods extends DKGAgentBase { markContextGraphMetaDirtyFromQuads: (quads) => { this.contextGraphMetaProjection.markDirtyFromQuads(quads); }, + writeLocks: this.writeLocks, + publicSnapshotStore: this.publicSnapshotStore, runtime: this.finalizationRuntime, }, ); diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index a3f1cc5c7a..503ac42641 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -45,11 +45,15 @@ import { compareMaterializedVersion, readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, materializedVersionQuad, withMaterializationLock, + withKeyedLocks, + swmKaWriteLockKey, KnowledgeAssetWorkspaceHeadCorruptError, + resolveKnowledgeAssetOperationPublicQuads, resolveKnowledgeAssetWorkspaceHead, workspacePublicQuadsDigest, type MaterializedVersion, type KnowledgeAssetWorkspaceHead, + type WorkspacePublicSnapshotStore, type KCMetadata, type KAMetadata, type OnChainProvenance, } from '@origintrail-official/dkg-publisher'; const DKG_NS = 'http://dkg.io/ontology/'; @@ -289,6 +293,8 @@ export interface FinalizationHandlerOptions { eventBus?: EventBus; resolveContextGraphOnChainId?: ResolveContextGraphOnChainId; markContextGraphMetaDirtyFromQuads?: MarkContextGraphMetaDirtyFromQuads; + writeLocks?: Map>; + publicSnapshotStore?: WorkspacePublicSnapshotStore; lifecycleLogOptions?: FinalizationLifecycleLogOptions; recoveryStore?: FinalizationRecoveryStore; runtime?: FinalizationRuntime; @@ -358,6 +364,8 @@ export class FinalizationHandler { private readonly eventBus: EventBus | undefined; private readonly resolveContextGraphOnChainId: ResolveContextGraphOnChainId | undefined; private readonly markContextGraphMetaDirtyFromQuads: MarkContextGraphMetaDirtyFromQuads | undefined; + private readonly writeLocks: Map>; + private readonly publicSnapshotStore: WorkspacePublicSnapshotStore | undefined; private readonly recovery: FinalizationRecovery; private readonly log = new Logger('FinalizationHandler'); private readonly lifecycle: FinalizationLifecycleLogger; @@ -416,6 +424,8 @@ export class FinalizationHandler { this.eventBus = options.eventBus; this.resolveContextGraphOnChainId = options.resolveContextGraphOnChainId; this.markContextGraphMetaDirtyFromQuads = options.markContextGraphMetaDirtyFromQuads; + this.writeLocks = options.writeLocks ?? new Map>(); + this.publicSnapshotStore = options.publicSnapshotStore; this.lifecycle = new FinalizationLifecycleLogger( this.log, options.runtime ?? options.lifecycleLogOptions, @@ -1074,6 +1084,18 @@ export class FinalizationHandler { expectedPublicQuadsDigest: head.publicQuadsDigest, subGraphName, }); + if (layerVerification.status !== 'verified') { + const snapshotVerification = await this.verifyImmutableGraphScopedSnapshot({ + contextGraphId, + scope, + head, + privateMerkleRoot, + expectedMerkleRoot: msg.kcMerkleRoot, + subGraphName, + ctx, + }); + if (snapshotVerification) layerVerification = snapshotVerification; + } if (layerVerification.status === 'count-mismatch') { this.log.warn( ctx, @@ -1168,6 +1190,16 @@ export class FinalizationHandler { subGraphName, }); if (metadataState === 'matching') { + await this.clearFinalizedGraphScopedSwm({ + contextGraphId, + scope, + expectedHead: head, + expectedMerkleRoot: msg.kcMerkleRoot, + privateMerkleRoot, + subGraphName, + source: 'finalization', + ctx, + }); this.markProcessed(dedupeKey); this.log.info(ctx, `Finalization: graph-scoped KA ${scope.ual} is already confirmed`); return 'already-confirmed'; @@ -1199,6 +1231,18 @@ export class FinalizationHandler { this.log.info(ctx, `Finalization: newer graph-scoped assertion already materialized for ${scope.ual}`); return 'already-confirmed'; } + if (outcome === 'applied') { + await this.clearFinalizedGraphScopedSwm({ + contextGraphId, + scope, + expectedHead: head, + expectedMerkleRoot: msg.kcMerkleRoot, + privateMerkleRoot, + subGraphName, + source: 'finalization', + ctx, + }); + } this.markProcessed(dedupeKey); this.log.info( @@ -1330,6 +1374,202 @@ export class FinalizationHandler { return { status: 'verified', graphUri, quads, merkleRoot }; } + /** + * Rebuild a finalized source from its immutable operation snapshot when the + * active SWM graph has already been drained. This keeps durable receipt/reorg + * recovery independent from retaining a second live copy of the payload. + */ + private async verifyImmutableGraphScopedSnapshot(input: { + contextGraphId: string; + scope: ReturnType; + head: KnowledgeAssetWorkspaceHead; + privateMerkleRoot?: Uint8Array; + expectedMerkleRoot: Uint8Array; + subGraphName?: string; + ctx: OperationContext; + }): Promise | undefined> { + let quads: Quad[]; + try { + const snapshot = await resolveKnowledgeAssetOperationPublicQuads({ + store: this.store, + graphManager: new GraphManager(this.store), + contextGraphId: input.contextGraphId, + shareOperationId: input.head.shareOperationId, + kaUal: input.scope.ual, + assertionVersion: input.scope.assertionVersion, + subGraphName: input.subGraphName, + publicSnapshotStore: this.publicSnapshotStore, + }); + quads = snapshot.quads.map((quad) => ({ ...quad, graph: '' })); + } catch (error) { + if (error instanceof StoreSchedulerBusyError) throw error; + this.log.warn( + input.ctx, + `Finalization: immutable graph-scoped snapshot is unavailable for ${input.scope.ual}: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + return undefined; + } + if ( + quads.length !== input.head.publicTripleCount + || workspacePublicQuadsDigest(quads) !== input.head.publicQuadsDigest + ) { + this.log.warn( + input.ctx, + `Finalization: immutable graph-scoped snapshot does not match the durable head for ` + + input.scope.ual, + ); + return undefined; + } + const merkleRoot = computeFlatKCRoot( + quads, + input.privateMerkleRoot ? [input.privateMerkleRoot] : [], + ); + if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { + this.log.warn( + input.ctx, + `Finalization: immutable graph-scoped snapshot does not match the chain root for ` + + input.scope.ual, + ); + return undefined; + } + return { + status: 'verified', + graphUri: input.head.assertionGraph, + quads, + merkleRoot, + }; + } + + /** + * Drain only the SWM assertion that was just proven and materialized. + * + * Source verification happens before the chain receipt is resolved, so the + * per-KA graph may have advanced by the time VM commits. Re-enter the exact + * lock used by every local, gossip, and catch-up SWM writer, re-read the + * durable head, and verify the graph again before deleting the payload. + * Keep the head and operation metadata: receipt recovery, reorg handling, + * and late publisher-policy upgrades still need that durable history. + */ + private async clearFinalizedGraphScopedSwm(input: { + contextGraphId: string; + scope: ReturnType; + expectedHead: KnowledgeAssetWorkspaceHead; + expectedMerkleRoot: Uint8Array; + privateMerkleRoot?: Uint8Array; + subGraphName?: string; + source: 'finalization' | 'chain-reconcile'; + ctx: OperationContext; + }): Promise<'cleared' | 'absent' | 'preserved'> { + const { + contextGraphId, + scope, + expectedHead, + expectedMerkleRoot, + privateMerkleRoot, + subGraphName, + source, + ctx, + } = input; + const lockKey = swmKaWriteLockKey(contextGraphId, subGraphName, scope.ual); + const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { + const graphManager = new GraphManager(this.store); + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager, + contextGraphId, + kaUal: scope.ual, + subGraphName, + }); + } catch (error) { + if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; + this.log.warn( + ctx, + `Finalization: preserving graph-scoped SWM for ${scope.ual}; ` + + `the current workspace head is corrupt: ${error.message}`, + ); + return 'preserved' as const; + } + if (!currentHead) return 'absent' as const; + + const sameAllowedPeers = [...currentHead.allowedPeers].sort().join('\0') + === [...expectedHead.allowedPeers].sort().join('\0'); + const sameHead = currentHead.kaUal === expectedHead.kaUal + && currentHead.assertionVersion === expectedHead.assertionVersion + && currentHead.assertionGraph === expectedHead.assertionGraph + && currentHead.publicQuadsDigest === expectedHead.publicQuadsDigest + && currentHead.publicTripleCount === expectedHead.publicTripleCount + && (currentHead.privateMerkleRoot?.toLowerCase() ?? undefined) + === (expectedHead.privateMerkleRoot?.toLowerCase() ?? undefined) + && currentHead.privateTripleCount === expectedHead.privateTripleCount + && currentHead.shareOperationId === expectedHead.shareOperationId + && currentHead.publisherPeerId === expectedHead.publisherPeerId + && currentHead.accessPolicy === expectedHead.accessPolicy + && sameAllowedPeers; + if (!sameHead) { + this.log.info( + ctx, + `Finalization: preserving newer graph-scoped SWM lifecycle for ${scope.ual}`, + ); + return 'preserved' as const; + } + + const verification = await this.verifyExactGraphScopedLayer({ + contextGraphId, + scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: expectedHead.publicTripleCount, + privateMerkleRoot, + expectedMerkleRoot, + expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, + subGraphName, + }); + if (verification.status === 'count-mismatch' && verification.actualCount === 0) { + return 'absent' as const; + } + if (verification.status !== 'verified') { + this.log.warn( + ctx, + `Finalization: preserving graph-scoped SWM for ${scope.ual}; ` + + `the current assertion no longer matches the finalized source (${verification.status})`, + ); + return 'preserved' as const; + } + + const replaced = await tryReplaceGraphAtomically( + this.store, + verification.graphUri, + [], + { source: 'agent.finalization.graphScopedSwmCleanup' }, + ); + if (!replaced) { + throw Object.assign( + new Error('Graph-scoped SWM finalization cleanup requires atomic graph replacement support'), + { code: 'SWM_ATOMIC_CLEANUP_UNSUPPORTED' }, + ); + } + return 'cleared' as const; + }); + + if (outcome === 'cleared') { + this.eventBus?.emit(DKGEvent.MEMORY_GRAPH_CHANGED, { + contextGraphId, + layers: ['swm'], + subGraphName, + operation: 'shared_working_memory_finalized', + source, + counts: { triples: expectedHead.publicTripleCount }, + }); + this.log.info( + ctx, + `Finalization: cleared finalized graph-scoped SWM assertion ${scope.ual}`, + ); + } + return outcome; + } + /** Recognize exact confirmed VM state from surviving immutable metadata. */ private async reconcileConfirmedGraphScopedVmWithoutWorkspaceHead(input: { contextGraphId: string; @@ -1532,6 +1772,25 @@ export class FinalizationHandler { this.log.warn(ctx, `Chain-reconcile: invalid private commitment for graph-scoped KA ${ual}`); return 'no-swm'; } + const clearMatchedWorkspaceHead = async (): Promise => { + if ( + !workspaceHead + || preserveNewerWorkspaceLifecycle + || workspaceHead.assertionVersion !== scope.assertionVersion + ) { + return; + } + await this.clearFinalizedGraphScopedSwm({ + contextGraphId, + scope, + expectedHead: workspaceHead, + expectedMerkleRoot: merkleRoot, + privateMerkleRoot, + subGraphName, + source: 'chain-reconcile', + ctx, + }); + }; const vmVerification = await this.verifyExactGraphScopedLayer({ contextGraphId, scope, @@ -1568,6 +1827,7 @@ export class FinalizationHandler { scope, materializedVersion, }); + await clearMatchedWorkspaceHead(); this.log.info(ctx, `Chain-reconcile: ${ual} already has exact VM content and metadata`); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } @@ -1636,11 +1896,12 @@ export class FinalizationHandler { ); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } + await clearMatchedWorkspaceHead(); this.log.info(ctx, `Chain-reconcile: exact VM graph already matches ${ual}; repaired metadata`); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } - const swmVerification = await this.verifyExactGraphScopedLayer({ + let swmVerification = await this.verifyExactGraphScopedLayer({ contextGraphId, scope, layer: MemoryLayer.SharedWorkingMemory, @@ -1652,6 +1913,23 @@ export class FinalizationHandler { : workspaceHead?.publicQuadsDigest, subGraphName, }); + if ( + swmVerification.status !== 'verified' + && trustedAssertionEvidence + && workspaceHead + && workspaceHead.assertionVersion === scope.assertionVersion + ) { + const snapshotVerification = await this.verifyImmutableGraphScopedSnapshot({ + contextGraphId, + scope, + head: workspaceHead, + privateMerkleRoot, + expectedMerkleRoot: merkleRoot, + subGraphName, + ctx, + }); + if (snapshotVerification) swmVerification = snapshotVerification; + } if (swmVerification.status === 'count-mismatch') { this.log.info( ctx, @@ -1715,6 +1993,7 @@ export class FinalizationHandler { ); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } + await clearMatchedWorkspaceHead(); this.log.info( ctx, `Chain-reconcile: promoted exact graph-scoped SWM assertion to VM for ${ual} (ka=${kaId})`, diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 91dcc004ea..c261912213 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -24,6 +24,8 @@ import { resolveKnowledgeAssetWorkspaceHead, storeKnowledgeAssetOperationPublicQuads, storeKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, } from '@origintrail-official/dkg-publisher'; import { FinalizationHandler } from '../src/finalization-handler.js'; import { @@ -355,7 +357,21 @@ describe('graph-scoped finalization handler', () => { }), CG, '12D3KooWPublisher'); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(0); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ + assertionVersion: VERSION, + shareOperationId: SHARE_ID, + }); + await expect(store.query( + `ASK { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { + ?operation ${JSON.stringify(SHARE_ID)} . + } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); const stale = await store.query( `ASK { GRAPH <${vmGraph}> { ?p ?o } }`, ); @@ -944,7 +960,7 @@ describe('graph-scoped finalization handler', () => { store.replaceGraphAndSubject = replaceGraphAndSubject; await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(0); await expect(store.query( `ASK { GRAPH <${metaGraph}> { <${UAL}> "${message.txHash}" } }`, )).resolves.toMatchObject({ type: 'boolean', value: true }); @@ -2053,6 +2069,69 @@ describe('graph-scoped finalization handler', () => { expect(currentHead?.assertionVersion).toBe('2'); }); + it('serializes finalized SWM cleanup with the shared per-KA writer lock', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + const writeLocks = new Map>(); + const lockingHandler = new FinalizationHandler(store, legacyFinalizationChain(), { + writeLocks, + }); + let releaseWriterLock!: () => void; + let markWriterLockAcquired!: () => void; + const writerLockAcquired = new Promise((resolve) => { + markWriterLockAcquired = resolve; + }); + const holdWriterLock = new Promise((resolve) => { + releaseWriterLock = resolve; + }); + const blocker = withKeyedLocks( + writeLocks, + [swmKaWriteLockKey(CG, undefined, UAL)], + async () => { + markWriterLockAcquired(); + await holdWriterLock; + }, + ); + await writerLockAcquired; + + let markVmCommitted!: () => void; + const vmCommitted = new Promise((resolve) => { + markVmCommitted = resolve; + }); + const replaceGraphAndSubject = store.replaceGraphAndSubject?.bind(store); + if (!replaceGraphAndSubject) throw new Error('Oxigraph replaceGraphAndSubject unavailable'); + store.replaceGraphAndSubject = async ( + graphUri, + quads, + metadataGraph, + subject, + metadata, + options, + ) => { + await replaceGraphAndSubject( + graphUri, + quads, + metadataGraph, + subject, + metadata, + options, + ); + if (graphUri === vmGraph) markVmCommitted(); + }; + + const finalization = lockingHandler.handleFinalizationMessage( + encodeFinalizationMessage(message), + CG, + ); + await vmCommitted; + expect(await store.countQuads(vmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(2); + + releaseWriterLock(); + await blocker; + await finalization; + expect(await store.countQuads(swmGraph)).toBe(0); + }); + it('rejects mixed graph-scope and legacy-root finalization envelopes', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage( @@ -2090,7 +2169,7 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(2); }); - it('verifies chain binding and exact private VM metadata without deleting unverified SWM', async () => { + it('verifies chain binding and exact private VM metadata before cleaning an exact SWM copy', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2135,7 +2214,7 @@ describe('graph-scoped finalization handler', () => { }, createOperationContext('system'))).resolves.toBe('already-confirmed'); expect(bindingVerified).toBe(true); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(0); }); it('recognizes only exact confirmed Verifiable Memory metadata after the workspace head is lost', async () => { @@ -2416,7 +2495,7 @@ describe('graph-scoped finalization handler', () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(0); const metaGraph = `did:dkg:context-graph:${CG}/_meta`; const materializedVersionPredicate = 'http://dkg.io/ontology/materializedVersion'; @@ -2533,9 +2612,10 @@ describe('graph-scoped finalization handler', () => { await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); const staged = await store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${swmGraph}> { ?s ?p ?o } }`, + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${vmGraph}> { ?s ?p ?o } }`, ); if (staged.type !== 'quads') throw new Error('expected staged SWM quads'); + await store.insert(staged.quads.map((quad) => ({ ...quad, graph: swmGraph }))); await storeKnowledgeAssetOperationPublicQuads({ store, graphManager, @@ -2647,9 +2727,10 @@ describe('graph-scoped finalization handler', () => { await store.deleteByPattern({ graph: metaGraph, subject: UAL }); const staged = await store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${swmGraph}> { ?s ?p ?o } }`, + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${vmGraph}> { ?s ?p ?o } }`, ); if (staged.type !== 'quads') throw new Error('expected staged SWM quads'); + await store.insert(staged.quads.map((quad) => ({ ...quad, graph: swmGraph }))); await storeKnowledgeAssetOperationPublicQuads({ store, graphManager, From 4a6245caeac2b234fc704002e6fe54829c64c997 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 10:58:02 +0200 Subject: [PATCH 02/48] fix(agent): defer finalized SWM cleanup under load --- packages/agent/src/dkg-agent-lifecycle.ts | 43 +- packages/agent/src/finalization-handler.ts | 555 +++++++++++++----- .../agent/src/sync/responder/graph-plan.ts | 56 +- .../ka-graph-finalization-handler.test.ts | 133 ++++- .../sync-responder-swm-meta-ceiling.test.ts | 7 +- .../test/sync-responder-swm-subgraphs.test.ts | 76 +++ packages/agent/test/workspace-ttl.test.ts | 13 +- packages/publisher/src/index.ts | 3 + .../publisher/src/workspace-resolution.ts | 37 +- 9 files changed, 749 insertions(+), 174 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 34b190fa1b..dbd75cf9f0 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -3114,15 +3114,13 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); } - // Start periodic shared memory cleanup - const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; - if (ttl > 0) { + // Start periodic SWM maintenance. TTL expiry may be disabled, but exact + // finalized graph-scoped copies still need bounded idle cleanup. + this.cleanupExpiredSharedMemory().catch(() => {}); + this.swmCleanupTimer = setInterval(() => { this.cleanupExpiredSharedMemory().catch(() => {}); - this.swmCleanupTimer = setInterval(() => { - this.cleanupExpiredSharedMemory().catch(() => {}); - }, SWM_CLEANUP_INTERVAL_MS); - if (this.swmCleanupTimer.unref) this.swmCleanupTimer.unref(); - } + }, SWM_CLEANUP_INTERVAL_MS); + if (this.swmCleanupTimer.unref) this.swmCleanupTimer.unref(); // OT-RFC-38 LU-6: periodic reconciler that ensures the local // node is subscribed in host-mode to every locally-known @@ -7549,18 +7547,14 @@ export class LifecycleSyncMethods extends DKGAgentBase { * and the next cleanup cycle without requiring a restart. */ setSharedMemoryTtlMs(this: DKGAgent, ttlMs: number): void { - const oldTtl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; (this.config as any).sharedMemoryTtlMs = ttlMs; - if (oldTtl <= 0 && ttlMs > 0 && !this.swmCleanupTimer) { + if (!this.swmCleanupTimer) { this.cleanupExpiredSharedMemory().catch(() => {}); this.swmCleanupTimer = setInterval(() => { this.cleanupExpiredSharedMemory().catch(() => {}); }, SWM_CLEANUP_INTERVAL_MS); if (this.swmCleanupTimer.unref) this.swmCleanupTimer.unref(); - } else if (ttlMs <= 0 && this.swmCleanupTimer) { - clearInterval(this.swmCleanupTimer); - this.swmCleanupTimer = null; } } @@ -7572,11 +7566,10 @@ export class LifecycleSyncMethods extends DKGAgentBase { */ async cleanupExpiredSharedMemory(this: DKGAgent): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; - if (ttl <= 0) return 0; - const ctx = createOperationContext('share'); - const cutoff = new Date(Date.now() - ttl).toISOString(); + const cutoff = ttl > 0 ? new Date(Date.now() - ttl).toISOString() : undefined; let totalDeleted = 0; + let finalizedCleanupBudget = 4; try { const graphManager = new GraphManager(this.store); @@ -7596,6 +7589,24 @@ export class LifecycleSyncMethods extends DKGAgentBase { // Each meta graph describes exactly one SWM data bucket: // `…/_shared_memory_meta` ↔ `…/_shared_memory` (root or per-subgraph). const wsGraph = wsMetaGraph.slice(0, -'_meta'.length); + if (finalizedCleanupBudget > 0) { + try { + const cleaned = await this.getOrCreateFinalizationHandler() + .cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: pid, + swmMetaGraph: wsMetaGraph, + maxCandidates: finalizedCleanupBudget, + }); + finalizedCleanupBudget -= cleaned; + } catch (error) { + this.log.warn( + ctx, + `Deferred finalized-SWM cleanup failed for ${wsMetaGraph}: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + } + } + if (!cutoff) continue; const expiredOps = await this.store.query( `SELECT ?op WHERE { diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index 503ac42641..c8f9402c22 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -26,6 +26,7 @@ import { tryReplaceGraphAndSubjectAtomically, StoreSchedulerBusyError, type GraphWriteGenSource, + type QueryOptions, type SharedMemoryResultBudget, type SwmKaGraphBound, type TripleStore, @@ -50,6 +51,9 @@ import { KnowledgeAssetWorkspaceHeadCorruptError, resolveKnowledgeAssetOperationPublicQuads, resolveKnowledgeAssetWorkspaceHead, + sameKnowledgeAssetWorkspaceHead, + workspaceKnowledgeAssetHeadSubject, + workspaceOperationSubject, workspacePublicQuadsDigest, type MaterializedVersion, type KnowledgeAssetWorkspaceHead, @@ -137,6 +141,7 @@ export const KEEP_ROOT_COPY_PREDICATE = `${DKG_NS}keepRootCopyOnLabel`; */ export const SWM_SNAPSHOT_MERKLE_ROOT_PREDICATE = `${DKG_NS}snapshotMerkleRoot`; export const SWM_SNAPSHOT_CONTENT_DIGEST_PREDICATE = `${DKG_NS}snapshotContentDigest`; +export const FINALIZED_SWM_CLEANUP_ROOT_PREDICATE = `${DKG_NS}finalizedSwmCleanupRoot`; /** * Resolves a local context-graph id (the topic/CG name used in gossip) to @@ -364,7 +369,7 @@ export class FinalizationHandler { private readonly eventBus: EventBus | undefined; private readonly resolveContextGraphOnChainId: ResolveContextGraphOnChainId | undefined; private readonly markContextGraphMetaDirtyFromQuads: MarkContextGraphMetaDirtyFromQuads | undefined; - private readonly writeLocks: Map>; + private readonly writeLocks: Map> | undefined; private readonly publicSnapshotStore: WorkspacePublicSnapshotStore | undefined; private readonly recovery: FinalizationRecovery; private readonly log = new Logger('FinalizationHandler'); @@ -424,7 +429,7 @@ export class FinalizationHandler { this.eventBus = options.eventBus; this.resolveContextGraphOnChainId = options.resolveContextGraphOnChainId; this.markContextGraphMetaDirtyFromQuads = options.markContextGraphMetaDirtyFromQuads; - this.writeLocks = options.writeLocks ?? new Map>(); + this.writeLocks = options.writeLocks; this.publicSnapshotStore = options.publicSnapshotStore; this.lifecycle = new FinalizationLifecycleLogger( this.log, @@ -1074,28 +1079,18 @@ export class FinalizationHandler { }); let layerVerification = vmVerification; if (layerVerification.status !== 'verified') { - layerVerification = await this.verifyExactGraphScopedLayer({ + layerVerification = await this.resolveVerifiedGraphScopedFinalizationSource({ contextGraphId, scope, - layer: MemoryLayer.SharedWorkingMemory, publicTripleCount, privateMerkleRoot, expectedMerkleRoot: msg.kcMerkleRoot, expectedPublicQuadsDigest: head.publicQuadsDigest, + expectedHead: head, subGraphName, + allowImmutableSnapshot: true, + ctx, }); - if (layerVerification.status !== 'verified') { - const snapshotVerification = await this.verifyImmutableGraphScopedSnapshot({ - contextGraphId, - scope, - head, - privateMerkleRoot, - expectedMerkleRoot: msg.kcMerkleRoot, - subGraphName, - ctx, - }); - if (snapshotVerification) layerVerification = snapshotVerification; - } if (layerVerification.status === 'count-mismatch') { this.log.warn( ctx, @@ -1190,14 +1185,12 @@ export class FinalizationHandler { subGraphName, }); if (metadataState === 'matching') { - await this.clearFinalizedGraphScopedSwm({ + await this.markFinalizedGraphScopedSwmForCleanup({ contextGraphId, scope, expectedHead: head, expectedMerkleRoot: msg.kcMerkleRoot, - privateMerkleRoot, subGraphName, - source: 'finalization', ctx, }); this.markProcessed(dedupeKey); @@ -1232,14 +1225,12 @@ export class FinalizationHandler { return 'already-confirmed'; } if (outcome === 'applied') { - await this.clearFinalizedGraphScopedSwm({ + await this.markFinalizedGraphScopedSwmForCleanup({ contextGraphId, scope, expectedHead: head, expectedMerkleRoot: msg.kcMerkleRoot, - privateMerkleRoot, subGraphName, - source: 'finalization', ctx, }); } @@ -1342,6 +1333,7 @@ export class FinalizationHandler { expectedMerkleRoot: Uint8Array; expectedPublicQuadsDigest?: string; subGraphName?: string; + queryOptions?: QueryOptions; }): Promise { const graphUri = knowledgeAssetLayerGraphUri( input.contextGraphId, @@ -1351,6 +1343,7 @@ export class FinalizationHandler { ); const result = await this.store.query( `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(graphUri)}> { ?s ?p ?o } }`, + input.queryOptions, ); const quads = result.type === 'quads' ? result.quads.map((quad) => ({ ...quad, graph: '' })) @@ -1382,85 +1375,372 @@ export class FinalizationHandler { private async verifyImmutableGraphScopedSnapshot(input: { contextGraphId: string; scope: ReturnType; - head: KnowledgeAssetWorkspaceHead; + expectedHead?: KnowledgeAssetWorkspaceHead; + publicTripleCount: number; + expectedPublicQuadsDigest?: string; privateMerkleRoot?: Uint8Array; expectedMerkleRoot: Uint8Array; subGraphName?: string; ctx: OperationContext; }): Promise | undefined> { - let quads: Quad[]; - try { - const snapshot = await resolveKnowledgeAssetOperationPublicQuads({ - store: this.store, - graphManager: new GraphManager(this.store), - contextGraphId: input.contextGraphId, - shareOperationId: input.head.shareOperationId, - kaUal: input.scope.ual, - assertionVersion: input.scope.assertionVersion, - subGraphName: input.subGraphName, - publicSnapshotStore: this.publicSnapshotStore, - }); - quads = snapshot.quads.map((quad) => ({ ...quad, graph: '' })); - } catch (error) { - if (error instanceof StoreSchedulerBusyError) throw error; - this.log.warn( - input.ctx, - `Finalization: immutable graph-scoped snapshot is unavailable for ${input.scope.ual}: ` - + `${error instanceof Error ? error.message : String(error)}`, + const graphManager = new GraphManager(this.store); + const shareOperationIds: string[] = []; + if (input.expectedHead) { + shareOperationIds.push(input.expectedHead.shareOperationId); + } else { + const metaGraph = graphManager.sharedMemoryMetaUri( + input.contextGraphId, + input.subGraphName, ); - return undefined; + const result = await this.store.query( + `SELECT DISTINCT ?shareId ?digest ?count WHERE { + GRAPH <${assertSafeIri(metaGraph)}> { + ?operation <${DKG_NS}contentScopeVersion> 2 ; + <${DKG_NS}kaUal> <${assertSafeIri(input.scope.ual)}> ; + <${DKG_NS}assertionVersion> ?version ; + <${DKG_NS}shareOperationId> ?shareId ; + <${DKG_NS}publicQuadsDigest> ?digest ; + <${DKG_NS}publicQuadsCount> ?count . + FILTER(STR(?version) = ${JSON.stringify(input.scope.assertionVersion)}) + } + } ORDER BY ?shareId LIMIT 16`, + ); + if (result.type === 'bindings') { + for (const row of result.bindings) { + const shareId = stripOptionalLiteral(row['shareId']); + const digest = stripOptionalLiteral(row['digest']); + const count = Number.parseInt(stripOptionalLiteral(row['count']) ?? '', 10); + if ( + shareId + && count === input.publicTripleCount + && ( + input.expectedPublicQuadsDigest === undefined + || digest === input.expectedPublicQuadsDigest + ) + ) { + shareOperationIds.push(shareId); + } + } + } } - if ( - quads.length !== input.head.publicTripleCount - || workspacePublicQuadsDigest(quads) !== input.head.publicQuadsDigest - ) { - this.log.warn( - input.ctx, - `Finalization: immutable graph-scoped snapshot does not match the durable head for ` - + input.scope.ual, + + for (const shareOperationId of [...new Set(shareOperationIds)]) { + let quads: Quad[]; + try { + const snapshot = await resolveKnowledgeAssetOperationPublicQuads({ + store: this.store, + graphManager, + contextGraphId: input.contextGraphId, + shareOperationId, + kaUal: input.scope.ual, + assertionVersion: input.scope.assertionVersion, + subGraphName: input.subGraphName, + publicSnapshotStore: this.publicSnapshotStore, + }); + quads = snapshot.quads.map((quad) => ({ ...quad, graph: '' })); + } catch (error) { + if (error instanceof StoreSchedulerBusyError) throw error; + continue; + } + if ( + quads.length !== input.publicTripleCount + || ( + input.expectedPublicQuadsDigest !== undefined + && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest + ) + ) { + continue; + } + const merkleRoot = computeFlatKCRoot( + quads, + input.privateMerkleRoot ? [input.privateMerkleRoot] : [], ); - return undefined; + if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) continue; + return { + status: 'verified', + graphUri: knowledgeAssetLayerGraphUri( + input.contextGraphId, + MemoryLayer.SharedWorkingMemory, + input.scope, + input.subGraphName, + ), + quads, + merkleRoot, + }; } - const merkleRoot = computeFlatKCRoot( - quads, - input.privateMerkleRoot ? [input.privateMerkleRoot] : [], + this.log.warn( + input.ctx, + `Finalization: no immutable graph-scoped snapshot matches ${input.scope.ual}`, ); - if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { - this.log.warn( + return undefined; + } + + /** + * Resolve one exact finalization source from the live SWM graph first and + * then, when receipt-backed recovery permits it, the immutable operation + * snapshot. Every finalization entrypoint shares this order and validation. + */ + private async resolveVerifiedGraphScopedFinalizationSource(input: { + contextGraphId: string; + scope: ReturnType; + publicTripleCount: number; + privateMerkleRoot?: Uint8Array; + expectedMerkleRoot: Uint8Array; + expectedPublicQuadsDigest?: string; + expectedHead?: KnowledgeAssetWorkspaceHead; + subGraphName?: string; + allowImmutableSnapshot: boolean; + ctx: OperationContext; + }): Promise { + const liveVerification = await this.verifyExactGraphScopedLayer({ + contextGraphId: input.contextGraphId, + scope: input.scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: input.publicTripleCount, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + expectedPublicQuadsDigest: input.expectedPublicQuadsDigest, + subGraphName: input.subGraphName, + }); + if (liveVerification.status === 'verified' || !input.allowImmutableSnapshot) { + return liveVerification; + } + const snapshotVerification = await this.verifyImmutableGraphScopedSnapshot({ + contextGraphId: input.contextGraphId, + scope: input.scope, + expectedHead: input.expectedHead, + publicTripleCount: input.publicTripleCount, + expectedPublicQuadsDigest: input.expectedPublicQuadsDigest, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + subGraphName: input.subGraphName, + ctx: input.ctx, + }); + return snapshotVerification ?? liveVerification; + } + + /** + * Mark one exact, already-materialized SWM head for deferred cleanup. + * + * The finalization path performs only this constant-size metadata write. + * Payload verification and deletion wait for the periodic maintenance lane, + * where store pressure is idle. A newer writer replaces the complete head + * subject under this same lock, which also removes the active-head marker. + * The immutable operation keeps the same marker until normal TTL retention + * removes it, so the finalized recovery snapshot cannot be re-advertised. + */ + private async markFinalizedGraphScopedSwmForCleanup(input: { + contextGraphId: string; + scope: ReturnType; + expectedHead: KnowledgeAssetWorkspaceHead; + expectedMerkleRoot: Uint8Array; + subGraphName?: string; + ctx: OperationContext; + }): Promise<'marked' | 'preserved'> { + if (!this.writeLocks) { + this.log.debug( input.ctx, - `Finalization: immutable graph-scoped snapshot does not match the chain root for ` - + input.scope.ual, + `Finalization: preserving graph-scoped SWM for ${input.scope.ual}; ` + + 'no shared SWM writer lock was provided', ); - return undefined; + return 'preserved'; } - return { - status: 'verified', - graphUri: input.head.assertionGraph, - quads, - merkleRoot, - }; + const lockKey = swmKaWriteLockKey( + input.contextGraphId, + input.subGraphName, + input.scope.ual, + ); + return withKeyedLocks(this.writeLocks, [lockKey], async () => { + const graphManager = new GraphManager(this.store); + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager, + contextGraphId: input.contextGraphId, + kaUal: input.scope.ual, + subGraphName: input.subGraphName, + }); + } catch (error) { + if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; + this.log.warn( + input.ctx, + `Finalization: preserving graph-scoped SWM for ${input.scope.ual}; ` + + `the current workspace head is corrupt: ${error.message}`, + ); + return 'preserved' as const; + } + if (!currentHead || !sameKnowledgeAssetWorkspaceHead(currentHead, input.expectedHead)) { + this.log.info( + input.ctx, + `Finalization: preserving newer graph-scoped SWM lifecycle for ${input.scope.ual}`, + ); + return 'preserved' as const; + } + const metaGraph = graphManager.sharedMemoryMetaUri( + input.contextGraphId, + input.subGraphName, + ); + const cleanupRoot = JSON.stringify( + ethers.hexlify(input.expectedMerkleRoot).toLowerCase(), + ); + await this.store.insert([ + { + subject: workspaceKnowledgeAssetHeadSubject(input.scope.ual), + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: cleanupRoot, + graph: metaGraph, + }, + { + subject: workspaceOperationSubject( + input.contextGraphId, + input.expectedHead.shareOperationId, + ), + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: cleanupRoot, + graph: metaGraph, + }, + ]); + return 'marked' as const; + }); } /** - * Drain only the SWM assertion that was just proven and materialized. + * Drain a bounded number of durable finalized-SWM markers only while the + * store scheduler reports no queued or in-flight work. * - * Source verification happens before the chain receipt is resolved, so the - * per-KA graph may have advanced by the time VM commits. Re-enter the exact - * lock used by every local, gossip, and catch-up SWM writer, re-read the - * durable head, and verify the graph again before deleting the payload. - * Keep the head and operation metadata: receipt recovery, reorg handling, - * and late publisher-policy upgrades still need that durable history. + * The marker survives restart, while replacing the SWM head for a newer + * assertion removes it automatically. All maintenance queries run in the + * background lane and the destructive step re-enters the canonical per-KA + * writer lock before checking the head, VM, SWM, and marker again. */ - private async clearFinalizedGraphScopedSwm(input: { + async cleanupFinalizedGraphScopedSwmWhenIdle(input: { + contextGraphId: string; + swmMetaGraph: string; + maxCandidates?: number; + }): Promise { + if (!this.writeLocks) return 0; + const pressure = this.store.getPressureSnapshot?.(); + if (pressure && ( + pressure.ackInflight > 0 + || (pressure.healthInflight ?? 0) > 0 + || pressure.normalInflight > 0 + || pressure.backgroundInflight > 0 + || pressure.ackQueued > 0 + || (pressure.healthQueued ?? 0) > 0 + || pressure.normalQueued > 0 + || pressure.backgroundQueued > 0 + )) { + return 0; + } + const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); + const background: QueryOptions = { + priority: 'background', + source: 'agent.finalization.graphScopedSwmCleanup.discover', + }; + const result = await this.store.query( + `SELECT DISTINCT ?head ?ual ?version ?root ?shareId ?subGraphName WHERE { + GRAPH <${assertSafeIri(input.swmMetaGraph)}> { + ?head <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; + <${DKG_NS}kaUal> ?ual ; + <${DKG_NS}assertionVersion> ?version ; + <${DKG_NS}shareOperationId> ?shareId ; + <${DKG_NS}assertionGraph> ?assertionGraph . + ?operation <${DKG_NS}shareOperationId> ?shareId ; + <${DKG_NS}kaUal> ?ual ; + <${DKG_NS}assertionVersion> ?version . + OPTIONAL { ?operation <${DKG_NS}subGraphName> ?subGraphName } + } + } ORDER BY ?head LIMIT ${limit}`, + background, + ); + if (result.type !== 'bindings') return 0; + + let cleared = 0; + for (const row of result.bindings) { + const currentPressure = this.store.getPressureSnapshot?.(); + if (currentPressure && ( + currentPressure.ackQueued > 0 + || (currentPressure.healthQueued ?? 0) > 0 + || currentPressure.normalQueued > 0 + )) { + break; + } + const ual = row['ual']; + const rawVersion = stripOptionalLiteral(row['version']); + const rawRoot = stripOptionalLiteral(row['root']); + const subGraphName = stripOptionalLiteral(row['subGraphName']); + if (!ual || !rawVersion || !/^\d+$/.test(rawVersion) || !rawRoot) continue; + let scope: ReturnType; + let expectedMerkleRoot: Uint8Array; + try { + scope = createGraphKnowledgeAssetScope(ual, BigInt(rawVersion)); + expectedMerkleRoot = ethers.getBytes(rawRoot); + } catch { + continue; + } + if ( + scope.ual !== ual + || expectedMerkleRoot.length !== 32 + || workspaceKnowledgeAssetHeadSubject(ual) !== row['head'] + ) { + continue; + } + const graphManager = new GraphManager(this.store); + let expectedHead: KnowledgeAssetWorkspaceHead | undefined; + try { + expectedHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager, + contextGraphId: input.contextGraphId, + kaUal: scope.ual, + subGraphName, + queryOptions: background, + }); + } catch (error) { + if (error instanceof StoreSchedulerBusyError) break; + continue; + } + if ( + !expectedHead + || expectedHead.assertionVersion !== scope.assertionVersion + || expectedHead.shareOperationId !== stripOptionalLiteral(row['shareId']) + ) { + continue; + } + let privateMerkleRoot: Uint8Array | undefined; + try { + privateMerkleRoot = expectedHead.privateMerkleRoot + ? ethers.getBytes(expectedHead.privateMerkleRoot) + : undefined; + } catch { + continue; + } + const outcome = await this.clearMarkedFinalizedGraphScopedSwm({ + contextGraphId: input.contextGraphId, + scope, + expectedHead, + expectedMerkleRoot, + privateMerkleRoot, + subGraphName, + ctx: createOperationContext('system'), + }); + if (outcome === 'cleared' || outcome === 'absent') cleared += 1; + } + return cleared; + } + + /** Atomically remove only the still-marked, still-exact active SWM lifecycle. */ + private async clearMarkedFinalizedGraphScopedSwm(input: { contextGraphId: string; scope: ReturnType; expectedHead: KnowledgeAssetWorkspaceHead; expectedMerkleRoot: Uint8Array; privateMerkleRoot?: Uint8Array; subGraphName?: string; - source: 'finalization' | 'chain-reconcile'; ctx: OperationContext; }): Promise<'cleared' | 'absent' | 'preserved'> { + if (!this.writeLocks) return 'preserved'; const { contextGraphId, scope, @@ -1468,12 +1748,15 @@ export class FinalizationHandler { expectedMerkleRoot, privateMerkleRoot, subGraphName, - source, ctx, } = input; const lockKey = swmKaWriteLockKey(contextGraphId, subGraphName, scope.ual); const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { const graphManager = new GraphManager(this.store); + const background: QueryOptions = { + priority: 'background', + source: 'agent.finalization.graphScopedSwmCleanup', + }; let currentHead: KnowledgeAssetWorkspaceHead | undefined; try { currentHead = await resolveKnowledgeAssetWorkspaceHead({ @@ -1482,6 +1765,7 @@ export class FinalizationHandler { contextGraphId, kaUal: scope.ual, subGraphName, + queryOptions: background, }); } catch (error) { if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; @@ -1493,22 +1777,7 @@ export class FinalizationHandler { return 'preserved' as const; } if (!currentHead) return 'absent' as const; - - const sameAllowedPeers = [...currentHead.allowedPeers].sort().join('\0') - === [...expectedHead.allowedPeers].sort().join('\0'); - const sameHead = currentHead.kaUal === expectedHead.kaUal - && currentHead.assertionVersion === expectedHead.assertionVersion - && currentHead.assertionGraph === expectedHead.assertionGraph - && currentHead.publicQuadsDigest === expectedHead.publicQuadsDigest - && currentHead.publicTripleCount === expectedHead.publicTripleCount - && (currentHead.privateMerkleRoot?.toLowerCase() ?? undefined) - === (expectedHead.privateMerkleRoot?.toLowerCase() ?? undefined) - && currentHead.privateTripleCount === expectedHead.privateTripleCount - && currentHead.shareOperationId === expectedHead.shareOperationId - && currentHead.publisherPeerId === expectedHead.publisherPeerId - && currentHead.accessPolicy === expectedHead.accessPolicy - && sameAllowedPeers; - if (!sameHead) { + if (!sameKnowledgeAssetWorkspaceHead(currentHead, expectedHead)) { this.log.info( ctx, `Finalization: preserving newer graph-scoped SWM lifecycle for ${scope.ual}`, @@ -1516,55 +1785,92 @@ export class FinalizationHandler { return 'preserved' as const; } - const verification = await this.verifyExactGraphScopedLayer({ + const metaGraph = graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); + const headSubject = workspaceKnowledgeAssetHeadSubject(scope.ual); + const cleanupRootObject = JSON.stringify( + ethers.hexlify(expectedMerkleRoot).toLowerCase(), + ); + const marker = await this.store.query( + `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` + + `<${assertSafeIri(headSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` + + `${cleanupRootObject} } }`, + background, + ); + if (marker.type !== 'boolean' || !marker.value) return 'preserved' as const; + + const vmVerification = await this.verifyExactGraphScopedLayer({ contextGraphId, scope, - layer: MemoryLayer.SharedWorkingMemory, + layer: MemoryLayer.VerifiableMemory, publicTripleCount: expectedHead.publicTripleCount, privateMerkleRoot, expectedMerkleRoot, expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, subGraphName, + queryOptions: background, }); - if (verification.status === 'count-mismatch' && verification.actualCount === 0) { - return 'absent' as const; + if (vmVerification.status !== 'verified') { + this.log.warn( + ctx, + `Finalization cleanup: preserving graph-scoped SWM for ${scope.ual}; ` + + `VM no longer matches the cleanup token (${vmVerification.status})`, + ); + return 'preserved' as const; } - if (verification.status !== 'verified') { + + const swmVerification = await this.verifyExactGraphScopedLayer({ + contextGraphId, + scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: expectedHead.publicTripleCount, + privateMerkleRoot, + expectedMerkleRoot, + expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, + subGraphName, + queryOptions: background, + }); + if ( + swmVerification.status !== 'verified' + && !(swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) + ) { this.log.warn( ctx, `Finalization: preserving graph-scoped SWM for ${scope.ual}; ` - + `the current assertion no longer matches the finalized source (${verification.status})`, + + `the current assertion no longer matches the finalized source (${swmVerification.status})`, ); return 'preserved' as const; } - const replaced = await tryReplaceGraphAtomically( + const replaced = await tryReplaceGraphAndSubjectAtomically( this.store, - verification.graphUri, + swmVerification.graphUri, + [], + metaGraph, + headSubject, [], - { source: 'agent.finalization.graphScopedSwmCleanup' }, + background, ); if (!replaced) { throw Object.assign( - new Error('Graph-scoped SWM finalization cleanup requires atomic graph replacement support'), + new Error('Graph-scoped SWM finalization cleanup requires atomic graph-and-head replacement support'), { code: 'SWM_ATOMIC_CLEANUP_UNSUPPORTED' }, ); } - return 'cleared' as const; + return swmVerification.status === 'verified' ? 'cleared' as const : 'absent' as const; }); - if (outcome === 'cleared') { + if (outcome === 'cleared' || outcome === 'absent') { this.eventBus?.emit(DKGEvent.MEMORY_GRAPH_CHANGED, { contextGraphId, layers: ['swm'], subGraphName, operation: 'shared_working_memory_finalized', - source, + source: 'background-cleanup', counts: { triples: expectedHead.publicTripleCount }, }); this.log.info( ctx, - `Finalization: cleared finalized graph-scoped SWM assertion ${scope.ual}`, + `Finalization cleanup: cleared finalized graph-scoped SWM assertion ${scope.ual}`, ); } return outcome; @@ -1772,7 +2078,7 @@ export class FinalizationHandler { this.log.warn(ctx, `Chain-reconcile: invalid private commitment for graph-scoped KA ${ual}`); return 'no-swm'; } - const clearMatchedWorkspaceHead = async (): Promise => { + const markMatchedWorkspaceHeadForCleanup = async (): Promise => { if ( !workspaceHead || preserveNewerWorkspaceLifecycle @@ -1780,14 +2086,12 @@ export class FinalizationHandler { ) { return; } - await this.clearFinalizedGraphScopedSwm({ + await this.markFinalizedGraphScopedSwmForCleanup({ contextGraphId, scope, expectedHead: workspaceHead, expectedMerkleRoot: merkleRoot, - privateMerkleRoot, subGraphName, - source: 'chain-reconcile', ctx, }); }; @@ -1827,7 +2131,7 @@ export class FinalizationHandler { scope, materializedVersion, }); - await clearMatchedWorkspaceHead(); + await markMatchedWorkspaceHeadForCleanup(); this.log.info(ctx, `Chain-reconcile: ${ual} already has exact VM content and metadata`); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } @@ -1896,40 +2200,27 @@ export class FinalizationHandler { ); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } - await clearMatchedWorkspaceHead(); + await markMatchedWorkspaceHeadForCleanup(); this.log.info(ctx, `Chain-reconcile: exact VM graph already matches ${ual}; repaired metadata`); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } - let swmVerification = await this.verifyExactGraphScopedLayer({ + const swmVerification = await this.resolveVerifiedGraphScopedFinalizationSource({ contextGraphId, scope, - layer: MemoryLayer.SharedWorkingMemory, publicTripleCount: head.publicTripleCount, privateMerkleRoot, expectedMerkleRoot: merkleRoot, expectedPublicQuadsDigest: trustedAssertionEvidence ? trustedAssertionEvidence.publicQuadsDigest : workspaceHead?.publicQuadsDigest, + expectedHead: workspaceHead?.assertionVersion === scope.assertionVersion + ? workspaceHead + : undefined, subGraphName, + allowImmutableSnapshot: trustedAssertionEvidence !== undefined, + ctx, }); - if ( - swmVerification.status !== 'verified' - && trustedAssertionEvidence - && workspaceHead - && workspaceHead.assertionVersion === scope.assertionVersion - ) { - const snapshotVerification = await this.verifyImmutableGraphScopedSnapshot({ - contextGraphId, - scope, - head: workspaceHead, - privateMerkleRoot, - expectedMerkleRoot: merkleRoot, - subGraphName, - ctx, - }); - if (snapshotVerification) swmVerification = snapshotVerification; - } if (swmVerification.status === 'count-mismatch') { this.log.info( ctx, @@ -1993,7 +2284,7 @@ export class FinalizationHandler { ); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } - await clearMatchedWorkspaceHead(); + await markMatchedWorkspaceHeadForCleanup(); this.log.info( ctx, `Chain-reconcile: promoted exact graph-scoped SWM assertion to VM for ${ual} (ka=${kaId})`, diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 9e59a89a26..077e964cf4 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -53,6 +53,7 @@ const DKG_CONTENT_SCOPE_VERSION = `${DKG}contentScopeVersion`; const DKG_KA_UAL = `${DKG}kaUal`; const DKG_ASSERTION_VERSION = `${DKG}assertionVersion`; const DKG_SHARE_OPERATION_ID = `${DKG}shareOperationId`; +const DKG_FINALIZED_SWM_CLEANUP_ROOT = `${DKG}finalizedSwmCleanupRoot`; const DKG_ASSERTION_GRAPH = `${DKG}assertionGraph`; const DKG_ASSERTION_NAME = `${DKG}assertionName`; const DKG_MEMORY_LAYER = `${DKG}memoryLayer`; @@ -2659,10 +2660,6 @@ function filterSwmMetaSnapshotRows( rows: readonly SyncRow[], cutoffIso: string | null, ): SyncRow[] { - if (cutoffIso == null) return [...rows].sort(compareRows); - const cutoffMs = Date.parse(cutoffIso); - if (!Number.isFinite(cutoffMs)) return []; - const bySubject = new Map(); for (const row of rows) { const bucket = bySubject.get(row.s) ?? []; @@ -2694,6 +2691,22 @@ function filterSwmMetaSnapshotRows( } return keys; }; + const blockedSubjects = new Set(); + const blockedTupleKeys = new Set(); + for (const [subject] of bySubject) { + if (objects(subject, DKG_FINALIZED_SWM_CLEANUP_ROOT).length === 0) continue; + blockedSubjects.add(subject); + for (const key of tupleKeys(subject)) blockedTupleKeys.add(key); + } + for (const [subject] of bySubject) { + if (tupleKeys(subject).some((key) => blockedTupleKeys.has(key))) { + blockedSubjects.add(subject); + } + } + const syncableRows = rows.filter((row) => !blockedSubjects.has(row.s)); + if (cutoffIso == null) return syncableRows.sort(compareRows); + const cutoffMs = Date.parse(cutoffIso); + if (!Number.isFinite(cutoffMs)) return []; const admitted = new Set(); const freshOperationKeys = new Set(); @@ -2710,12 +2723,13 @@ function filterSwmMetaSnapshotRows( if (tupleKeys(subject).some((key) => freshOperationKeys.has(key))) admitted.add(subject); } - return rows.filter((row) => admitted.has(row.s)).sort(compareRows); + return syncableRows.filter((row) => admitted.has(row.s)).sort(compareRows); } /** - * Legacy UNFILTERED store-paged compatibility path (cutoffIso == null sessions - * only). The former TTL variant of this query — DISTINCT + a six-predicate + * Legacy TTL-unfiltered store-paged compatibility path (cutoffIso == null + * sessions only). Finalized cleanup-marked lifecycles are still excluded from + * synchronization. The former TTL variant of this query — DISTINCT + a six-predicate * UNION join + global `ORDER BY ?g ?s ?p ?o` re-evaluated with a growing * OFFSET per page over a mutable graph family — was the #1847 store-melter and * is deliberately DELETED, not gated: TTL-filtered sessions page from the @@ -2734,12 +2748,22 @@ async function readSwmMetaRowsPage( const swmMetaValues = graphValues(swmMetaGraphs); if (!swmMetaValues) return []; // sparql-scan-allow: R2 -- ?g is bound by a finite VALUES list of pre-admitted SWM meta graph IRIs - // sparql-scan-allow: R3 -- pre-existing legacy (cutoff-less) compatibility lane, unchanged behavior; TTL sessions page from the session plan instead (#1847) + // sparql-scan-allow: R3 -- legacy cutoff-less compatibility lane with only finalized-lifecycle exclusion; TTL sessions page from the session plan instead (#1847) const res = await store.query(` SELECT DISTINCT ?g ?s ?p ?o WHERE { VALUES ?g { ${swmMetaValues} } GRAPH ?g { ?s ?p ?o . + FILTER NOT EXISTS { ?s <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot } + FILTER NOT EXISTS { + ?blockedHead <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; + <${DKG_KA_UAL}> ?blockedUal ; + <${DKG_ASSERTION_VERSION}> ?blockedVersion ; + <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . + ?s <${DKG_KA_UAL}> ?blockedUal ; + <${DKG_ASSERTION_VERSION}> ?blockedVersion ; + <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . + } } } ORDER BY ?g ?s ?p ?o @@ -2838,6 +2862,15 @@ async function readFreshSwmMetaSubjects( SELECT DISTINCT ?s WHERE { GRAPH <${assertSafeIri(graph)}> { ?s <${DKG_PUBLISHED_AT}> ?ts . + FILTER NOT EXISTS { + ?blockedHead <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; + <${DKG_KA_UAL}> ?blockedUal ; + <${DKG_ASSERTION_VERSION}> ?blockedVersion ; + <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . + ?s <${DKG_KA_UAL}> ?blockedUal ; + <${DKG_ASSERTION_VERSION}> ?blockedVersion ; + <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . + } ${cutoffFilter} } } @@ -2850,6 +2883,13 @@ async function readFreshSwmMetaSubjects( <${DKG_KA_UAL}> ?headUal ; <${DKG_ASSERTION_VERSION}> ?headVersion ; <${DKG_SHARE_OPERATION_ID}> ?shareId . + FILTER NOT EXISTS { ?s <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot } + FILTER NOT EXISTS { + ?blockedHead <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; + <${DKG_KA_UAL}> ?headUal ; + <${DKG_ASSERTION_VERSION}> ?headVersion ; + <${DKG_SHARE_OPERATION_ID}> ?shareId . + } ?headOperation <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_WORKSPACE_OPERATION}> ; <${DKG_CONTENT_SCOPE_VERSION}> ${GRAPH_KA_CONTENT_SCOPE_VERSION} ; <${DKG_KA_UAL}> ?headUal ; diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index c261912213..4f2e31ff0a 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -26,8 +26,12 @@ import { storeKnowledgeAssetWorkspaceHead, swmKaWriteLockKey, withKeyedLocks, + workspaceOperationSubject, } from '@origintrail-official/dkg-publisher'; -import { FinalizationHandler } from '../src/finalization-handler.js'; +import { + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FinalizationHandler, +} from '../src/finalization-handler.js'; import { openSqliteFinalizationRecoveryStore, type SqliteFinalizationRecoveryStore, @@ -171,13 +175,26 @@ describe('graph-scoped finalization handler', () => { let store: OxigraphStore; let graphManager: GraphManager; let handler: FinalizationHandler; + let writeLocks: Map>; beforeEach(() => { store = new OxigraphStore(); graphManager = new GraphManager(store); - handler = new FinalizationHandler(store, legacyFinalizationChain()); + writeLocks = new Map>(); + handler = new FinalizationHandler(store, legacyFinalizationChain(), { writeLocks }); }); + async function drainFinalizedSwm( + target = handler, + subGraphName?: string, + ): Promise { + return target.cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG, subGraphName), + maxCandidates: 16, + }); + } + async function stageGraph(durableAccess?: { accessPolicy: 'ownerOnly' | 'allowList'; allowedPeers?: string[]; @@ -357,16 +374,15 @@ describe('graph-scoped finalization handler', () => { }), CG, '12D3KooWPublisher'); expect(await store.countQuads(vmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); await expect(resolveKnowledgeAssetWorkspaceHead({ store, graphManager, contextGraphId: CG, kaUal: UAL, - })).resolves.toMatchObject({ - assertionVersion: VERSION, - shareOperationId: SHARE_ID, - }); + })).resolves.toBeUndefined(); await expect(store.query( `ASK { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { ?operation ${JSON.stringify(SHARE_ID)} . @@ -960,6 +976,7 @@ describe('graph-scoped finalization handler', () => { store.replaceGraphAndSubject = replaceGraphAndSubject; await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); expect(await store.countQuads(vmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); await expect(store.query( `ASK { GRAPH <${metaGraph}> { <${UAL}> "${message.txHash}" } }`, @@ -2129,6 +2146,108 @@ describe('graph-scoped finalization handler', () => { releaseWriterLock(); await blocker; await finalization; + expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm(lockingHandler)).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + }); + + it('preserves finalized SWM when the handler has no shared writer lock', async () => { + const { message, swmGraph } = await stageGraph(); + const uncoordinated = new FinalizationHandler(store, legacyFinalizationChain()); + + await uncoordinated.handleFinalizationMessage( + encodeFinalizationMessage(message), + CG, + ); + + expect(await uncoordinated.cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + })).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); + }); + + it('defers durable cleanup while the store is busy and resumes after restart when idle', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + let busy = true; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + + busy = false; + const restarted = new FinalizationHandler(store, legacyFinalizationChain(), { + writeLocks, + }); + expect(await drainFinalizedSwm(restarted)).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toBeUndefined(); + const immutableSnapshotBlockedFromSync = await store.query( + `ASK { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { ` + + `<${workspaceOperationSubject(CG, SHARE_ID)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + ); + expect(immutableSnapshotBlockedFromSync).toMatchObject({ + type: 'boolean', + value: true, + }); + }); + + it('repairs VM from the immutable operation snapshot after deferred SWM cleanup', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + const internals = handler as unknown as { + verifyChainCgBinding: (kaId: bigint, cgId: string) => Promise; + }; + internals.verifyChainCgBinding = async () => true; + + await store.dropGraph(vmGraph); + await store.deleteByPattern({ + graph: graphManager.metaGraphUri(CG), + subject: UAL, + }); + + await expect(handler.handleChainReconciledKC({ + contextGraphId: CG, + onChainCgId: '42', + ual: UAL, + merkleRoot: message.kcMerkleRoot, + publisherAddress: PUBLISHER, + kaId: PACKED_KA_ID, + versionBlock: 123, + authorAddress: AUTHOR, + trustedAssertionEvidence: trustedRecoveryEvidence(message), + }, createOperationContext('system'))).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); expect(await store.countQuads(swmGraph)).toBe(0); }); @@ -2214,6 +2333,7 @@ describe('graph-scoped finalization handler', () => { }, createOperationContext('system'))).resolves.toBe('already-confirmed'); expect(bindingVerified).toBe(true); expect(await store.countQuads(vmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); }); @@ -2495,6 +2615,7 @@ describe('graph-scoped finalization handler', () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); expect(await store.countQuads(vmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); const metaGraph = `did:dkg:context-graph:${CG}/_meta`; diff --git a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts index 5f0aebf3e9..14d246bc0a 100644 --- a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts +++ b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts @@ -639,7 +639,7 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { await store.close(); }, 120_000); - it('legacy cutoff-less sessions keep the unfiltered store-paged compatibility fallback', async () => { + it('legacy cutoff-less sessions keep TTL joins out of the store-paged compatibility fallback', async () => { const cgId = 'meta-ceiling-legacy'; const metaGraph = `did:dkg:context-graph:${cgId}/_shared_memory_meta`; const store = new OxigraphStore(); @@ -658,9 +658,10 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { normalized.includes('ORDER BY ?g ?s ?p ?o') && /OFFSET \d+/.test(normalized) ) { - // The legacy paged query must never carry the TTL join. + // The legacy paged query must never carry the TTL join. The only + // filters allowed here exclude finalized cleanup-marked lifecycles. expect(normalized).not.toContain('publishedAt'); - expect(normalized).not.toContain('FILTER'); + expect(normalized).toContain('finalizedSwmCleanupRoot'); legacyPagedQueries += 1; } return originalQuery(sparql, options as never); diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index 6fe23b9c7e..029747a129 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -408,6 +408,82 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { expect(out).not.toContain('urn:dkg:share:devnet-test:stale-rootless'); await storeTtl.close(); }); + + it.each([0, 5_000])( + 'does not advertise finalized SWM marked for deferred cleanup (ttl=%s)', + async (sharedMemoryTtlMs) => { + const markedStore = new OxigraphStore(); + const publishedAt = new Date(Date.now() - 1_000).toISOString(); + const ual = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/9'; + const opId = 'finalized-deferred-cleanup'; + const op = `urn:dkg:share:${CG_ID}:${opId}`; + const head = `${ual}#dkg-swm-head`; + const tupleRows = (subject: string) => [ + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}contentScopeVersion`, object: '"2"^^' }, + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}kaUal`, object: ual }, + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}assertionVersion`, object: '"9"^^' }, + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}shareOperationId`, object: `"${opId}"` }, + ]; + await markedStore.insert([ + { graph: ROOT_SWM_META, subject: op, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + ...tupleRows(op), + ...tupleRows(head), + { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}assertionGraph`, object: `${ROOT_SWM}/0x00000000000000000000000000000000000000ab/9` }, + { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + ]); + const markedCap = captureHandler(); + registerSyncHandler({ + register: markedCap.register, + protocolSync: '/origintrail/dkg/sync/1.0.0', + syncDeniedResponse: 'sync-denied', + syncPageSize: 5000, + sharedMemoryTtlMs, + store: markedStore, + peerId: 'self-peer', + parseSyncRequest: (data) => JSON.parse(new TextDecoder().decode(data)) as SyncRequestEnvelope, + authorizeSyncRequest: async () => true, + logWarn: noopLog, + logDebug: noopLog, + }); + + const out = await markedCap.invoke({ + contextGraphId: CG_ID, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + + expect(out).not.toContain(head); + expect(out).not.toContain(op); + expect(out).not.toContain('finalizedSwmCleanupRoot'); + + // After idle cleanup removes the active head, the immutable operation + // keeps the marker for recovery but must remain outside sync. + await markedStore.deleteByPattern({ + graph: ROOT_SWM_META, + subject: head, + }); + await markedStore.insert([{ + graph: ROOT_SWM_META, + subject: op, + predicate: `${DKG_NS}finalizedSwmCleanupRoot`, + object: `"0x${'ab'.repeat(32)}"`, + }]); + const afterCleanup = await markedCap.invoke({ + contextGraphId: CG_ID, + syncSessionId: `finalized-cleanup-drained-${sharedMemoryTtlMs}`, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + expect(afterCleanup).not.toContain(op); + expect(afterCleanup).not.toContain('finalizedSwmCleanupRoot'); + await markedStore.close(); + }, + ); }); // Codex review on #885 — URI shape alone is NOT a sufficient CG diff --git a/packages/agent/test/workspace-ttl.test.ts b/packages/agent/test/workspace-ttl.test.ts index 31263c7aba..bef61bb4a6 100644 --- a/packages/agent/test/workspace-ttl.test.ts +++ b/packages/agent/test/workspace-ttl.test.ts @@ -89,14 +89,14 @@ describe('Workspace TTL', () => { }, 20000); }); -describe('setSharedMemoryTtlMs timer lifecycle', () => { +describe('setSharedMemoryTtlMs maintenance timer lifecycle', () => { let node: DKGAgent; afterAll(async () => { try { await node?.stop(); } catch {} }); - it('starts cleanup timer when TTL transitions from 0 to positive', async () => { + it('keeps finalized-SWM maintenance active when TTL expiry is disabled', async () => { node = await DKGAgent.create({ kaNumberAllocator: makeTestKaNumberAllocator(), name: 'TtlLifecycleNode', @@ -108,16 +108,17 @@ describe('setSharedMemoryTtlMs timer lifecycle', () => { await node.start(); await sleep(300); - // Timer should not be running (TTL=0) - expect((node as any).swmCleanupTimer).toBeNull(); + // Finalized graph-scoped SWM cleanup remains active even when ordinary + // workspace TTL expiry is disabled. + expect((node as any).swmCleanupTimer).not.toBeNull(); // Enable TTL at runtime node.setSharedMemoryTtlMs(60_000); expect((node as any).swmCleanupTimer).not.toBeNull(); - // Disable again + // Disabling TTL expiry must not disable finalized-SWM maintenance. node.setSharedMemoryTtlMs(0); - expect((node as any).swmCleanupTimer).toBeNull(); + expect((node as any).swmCleanupTimer).not.toBeNull(); }, 10000); }); diff --git a/packages/publisher/src/index.ts b/packages/publisher/src/index.ts index 61fa9e63af..bf9f47455a 100644 --- a/packages/publisher/src/index.ts +++ b/packages/publisher/src/index.ts @@ -44,10 +44,13 @@ export { } from './catalog-trust.js'; export { resolveKnowledgeAssetWorkspaceHead, + sameKnowledgeAssetWorkspaceHead, resolveKnowledgeAssetOperationPublicQuads, resolveLiftWorkspaceSlice, storeKnowledgeAssetWorkspaceHead, storeKnowledgeAssetOperationPublicQuads, + workspaceKnowledgeAssetHeadSubject, + workspaceOperationSubject, KnowledgeAssetOperationPublicSnapshotNotFoundError, KnowledgeAssetWorkspaceHeadCorruptError, type KnowledgeAssetWorkspaceHead, diff --git a/packages/publisher/src/workspace-resolution.ts b/packages/publisher/src/workspace-resolution.ts index b2326ce404..1f992be921 100644 --- a/packages/publisher/src/workspace-resolution.ts +++ b/packages/publisher/src/workspace-resolution.ts @@ -1,4 +1,4 @@ -import type { Quad, TripleStore } from '@origintrail-official/dkg-storage'; +import type { Quad, QueryOptions, TripleStore } from '@origintrail-official/dkg-storage'; import { GraphManager, PrivateContentStore } from '@origintrail-official/dkg-storage'; import { GRAPH_KA_CONTENT_SCOPE_VERSION, @@ -101,6 +101,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { contextGraphId: string; kaUal: string; subGraphName?: string; + queryOptions?: QueryOptions; }): Promise { const scope = createGraphKnowledgeAssetScope(params.kaUal, 1); const subGraphName = normalizeOptionalSubGraphName(params.subGraphName); @@ -128,6 +129,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { OPTIONAL { ?operation <${DKG}accessPolicy> ?accessPolicy } } } LIMIT 1`, + params.queryOptions, ); if (result.type !== 'bindings') { throw new Error( @@ -138,6 +140,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { const existence = await params.store.query( `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` + `<${assertSafeIri(subject)}> ?predicate ?object } }`, + params.queryOptions, ); if (existence.type !== 'boolean') { throw new Error( @@ -240,6 +243,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { const peersResult = await params.store.query( `SELECT ?peer WHERE { GRAPH <${assertSafeIri(metaGraph)}> { ` + `<${assertSafeIri(expectedOperationSubject)}> <${DKG}allowedPeer> ?peer } }`, + params.queryOptions, ); if (peersResult.type !== 'bindings') { throw new Error( @@ -272,6 +276,33 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { }; } +/** + * Canonical identity comparison for the current graph-scoped SWM head. + * + * Cleanup and writers share this helper so adding a lifecycle field cannot + * silently make a destructive cleanup compare less state than the workspace + * model owns. + */ +export function sameKnowledgeAssetWorkspaceHead( + left: KnowledgeAssetWorkspaceHead, + right: KnowledgeAssetWorkspaceHead, +): boolean { + const sameAllowedPeers = [...left.allowedPeers].sort().join('\0') + === [...right.allowedPeers].sort().join('\0'); + return left.kaUal === right.kaUal + && left.assertionVersion === right.assertionVersion + && left.assertionGraph === right.assertionGraph + && left.publicQuadsDigest === right.publicQuadsDigest + && left.publicTripleCount === right.publicTripleCount + && (left.privateMerkleRoot?.toLowerCase() ?? undefined) + === (right.privateMerkleRoot?.toLowerCase() ?? undefined) + && left.privateTripleCount === right.privateTripleCount + && left.shareOperationId === right.shareOperationId + && left.publisherPeerId === right.publisherPeerId + && left.accessPolicy === right.accessPolicy + && sameAllowedPeers; +} + /** Replace the durable current-assertion pointer after data and snapshot land. */ export async function storeKnowledgeAssetWorkspaceHead(params: { store: TripleStore; @@ -1124,7 +1155,7 @@ function normalizeOptionalSubGraphName(subGraphName: string | undefined): string return normalized; } -function workspaceOperationSubject(contextGraphId: string, shareOperationId: string): string { +export function workspaceOperationSubject(contextGraphId: string, shareOperationId: string): string { const normalizedContextGraphId = safeWorkspaceIdPart(contextGraphId, 'contextGraphId'); const normalizedShareOperationId = safeWorkspaceIdPart(shareOperationId, 'shareOperationId'); const subject = `urn:dkg:share:${normalizedContextGraphId}:${normalizedShareOperationId}`; @@ -1132,7 +1163,7 @@ function workspaceOperationSubject(contextGraphId: string, shareOperationId: str return subject; } -function workspaceKnowledgeAssetHeadSubject(kaUal: string): string { +export function workspaceKnowledgeAssetHeadSubject(kaUal: string): string { const scope = createGraphKnowledgeAssetScope(kaUal, 1); const subject = `${scope.ual}#dkg-swm-head`; assertSafeIri(subject); From 3a5ab91d36062bd061bc98fb187748dd17b57812 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 15:14:32 +0200 Subject: [PATCH 03/48] fix(agent): preserve finalized SWM cleanup through recovery --- packages/agent/src/dkg-agent-constants.ts | 2 + packages/agent/src/dkg-agent-lifecycle.ts | 121 ++++++---- packages/agent/src/finalization-handler.ts | 3 +- .../src/sync/graph-scoped-swm-recovery.ts | 17 ++ .../src/sync/requester/shared-memory-sync.ts | 43 +++- .../agent/src/sync/requester/swm-recovery.ts | 20 +- .../requester/swm-snapshot-materializer.ts | 209 ++++++++++++++---- ...wm-public-snapshot-materialization.test.ts | 24 +- .../test/swm-snapshot-materializer.test.ts | 94 +++++++- .../test/sync-requester-priority.test.ts | 10 + 10 files changed, 415 insertions(+), 128 deletions(-) diff --git a/packages/agent/src/dkg-agent-constants.ts b/packages/agent/src/dkg-agent-constants.ts index e58181cfea..b2fc60750a 100644 --- a/packages/agent/src/dkg-agent-constants.ts +++ b/packages/agent/src/dkg-agent-constants.ts @@ -134,6 +134,8 @@ export const SYNC_MIN_GRAPH_BUDGET_MS = 10_000; export const DEBUG_SYNC_PROGRESS = process.env.DKG_DEBUG_SYNC_PROGRESS === '1'; export const DEFAULT_SWM_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days export const SWM_CLEANUP_INTERVAL_MS = 15 * 60 * 1000; // run cleanup every 15 minutes +export const FINALIZED_SWM_CLEANUP_ROOT_PREDICATE = + 'http://dkg.io/ontology/finalizedSwmCleanupRoot'; export const SYNC_DENIED_RESPONSE = '__DKG_SYNC_DENIED__'; // ── Gossip reconnect ────────────────────────────────────────────────── diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index dbd75cf9f0..97d10f3aac 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -138,6 +138,8 @@ import { TripleStoreAsyncPromoteQueue, FileWorkspacePublicSnapshotStore, parseWorkspacePublicSnapshotNQuads, + swmKaWriteLockKey, + withKeyedLocks, type AsyncPromoteQueue, type AsyncPromoteQueueConfig, type PromoteJob, type PromoteListFilter, wrapAsRpcPreconditionIfApplicable, @@ -264,7 +266,10 @@ import { } from './sync/requester/durable-sync.js'; import { resolveSyncAgentsMeta, shouldWithholdAgentsDurableMeta } from './sync/agents-meta-policy.js'; import { runSharedMemorySync, sharedMemoryOwnershipKeyFromGraph } from './sync/requester/shared-memory-sync.js'; -import { createSharedMemorySnapshotMaterializer } from './sync/requester/swm-snapshot-materializer.js'; +import { + createSharedMemorySnapshotMaterializer, + replaceGraphScopedSwmHeadMetadata, +} from './sync/requester/swm-snapshot-materializer.js'; import { runOrderedContextGraphSyncs, type ContextGraphSyncWork, @@ -886,6 +891,7 @@ type RecoverContextGraphSwmOptions = Parameters[0 interface RecoverContextGraphSwmFromPeerDependencies { store: TripleStore; + writeLocks: Map>; listSubGraphs: (contextGraphId: string) => ReturnType; createContextGraphSyncDeadline: (remainingContextGraphs: number) => number; fetchSyncPages: RecoverContextGraphSwmOptions['fetchSyncPages']; @@ -5211,6 +5217,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { const recoverPrivateContextGraph = (contextGraphId: string) => runRecoverContextGraphSwmFromPeer( { store: this.store, + writeLocks: this.writeLocks, listSubGraphs: (id) => this.listSubGraphs(id), createContextGraphSyncDeadline: (remaining) => createContextGraphSyncDeadline({ remainingContextGraphs: remaining, @@ -5415,7 +5422,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); } - return runOrderedContextGraphSyncs({ + const summary = await runOrderedContextGraphSyncs({ work, priorities: this.config.syncContextGraphPriorities, emptyResult: emptySharedMemorySyncResult, @@ -5440,6 +5447,23 @@ export class LifecycleSyncMethods extends DKGAgentBase { `Deferring ${item.lane} at CG ${item.contextGraphId} due to local backpressure: ${error.message}`, ), }); + if ( + work.length > 0 + && summary.failedPhases === 0 + && typeof this.cleanupExpiredSharedMemory === 'function' + ) { + // A completed catch-up is the first reliable low-pressure boundary + // after a large synchronized write. Drain finalized duplicates here, + // before reporting the job complete, in small background-priority + // batches that stop as soon as normal work appears. The periodic + // maintenance timer remains the restart/failure backstop. + await this.cleanupExpiredSharedMemory({ + finalizedOnly: true, + contextGraphIds: work.map((item) => item.contextGraphId), + finalizedCleanupBudget: 64, + }); + } + return summary; }; return runSyncSingleFlight(this, singleFlightKey, runSync); @@ -5470,6 +5494,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { () => runRecoverContextGraphSwmFromPeer( { store: this.store, + writeLocks: this.writeLocks, listSubGraphs: (id) => this.listSubGraphs(id), createContextGraphSyncDeadline: (remaining) => createContextGraphSyncDeadline({ remainingContextGraphs: remaining, @@ -7559,21 +7584,35 @@ export class LifecycleSyncMethods extends DKGAgentBase { } /** - * Remove expired shared memory operations and their data. - * Queries SWM meta for operations with publishedAt older than the TTL, - * deletes the corresponding triples from shared memory and SWM meta, - * and removes the root entities from workspaceOwnedEntities. + * Remove expired shared-memory operations and, at safe low-pressure + * boundaries, exact finalized SWM lifecycles retained for deferred cleanup. + * For stale operations it deletes the corresponding triples from shared + * memory and SWM meta, and removes the root entities from + * workspaceOwnedEntities. */ - async cleanupExpiredSharedMemory(this: DKGAgent): Promise { + async cleanupExpiredSharedMemory(this: DKGAgent, options?: { + finalizedOnly?: boolean; + contextGraphIds?: readonly string[]; + finalizedCleanupBudget?: number; + }): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; const ctx = createOperationContext('share'); - const cutoff = ttl > 0 ? new Date(Date.now() - ttl).toISOString() : undefined; + const cutoff = !options?.finalizedOnly && ttl > 0 + ? new Date(Date.now() - ttl).toISOString() + : undefined; let totalDeleted = 0; - let finalizedCleanupBudget = 4; + let finalizedCleanupBudget = Math.max( + 0, + Math.floor(options?.finalizedCleanupBudget ?? 4), + ); try { const graphManager = new GraphManager(this.store); - const contextGraphs = await graphManager.listContextGraphs(); + const requestedContextGraphs = options?.contextGraphIds + ? new Set(options.contextGraphIds) + : undefined; + const contextGraphs = (await graphManager.listContextGraphs()) + .filter((contextGraphId) => !requestedContextGraphs || requestedContextGraphs.has(contextGraphId)); for (const pid of contextGraphs) { let graphDeleted = 0; @@ -7591,13 +7630,20 @@ export class LifecycleSyncMethods extends DKGAgentBase { const wsGraph = wsMetaGraph.slice(0, -'_meta'.length); if (finalizedCleanupBudget > 0) { try { - const cleaned = await this.getOrCreateFinalizationHandler() - .cleanupFinalizedGraphScopedSwmWhenIdle({ - contextGraphId: pid, - swmMetaGraph: wsMetaGraph, - maxCandidates: finalizedCleanupBudget, - }); - finalizedCleanupBudget -= cleaned; + while (finalizedCleanupBudget > 0) { + const batchSize = Math.min(4, finalizedCleanupBudget); + const cleaned = await this.getOrCreateFinalizationHandler() + .cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: pid, + swmMetaGraph: wsMetaGraph, + maxCandidates: batchSize, + }); + finalizedCleanupBudget -= cleaned; + if (cleaned < batchSize) break; + // Yield between bounded batches so queued foreground work can + // arrive and trip the handler's pressure gate. + await new Promise((resolve) => setImmediate(resolve)); + } } catch (error) { this.log.warn( ctx, @@ -7899,39 +7945,16 @@ async function runRecoverContextGraphSwmFromPeer( }, replaceMetaForGraphAssets: async (assets) => { for (const asset of assets) { - const linkedOperations = await dependencies.store.query( - `SELECT DISTINCT ?op WHERE { GRAPH <${assertSafeIri(asset.metaGraph)}> { ` + - `<${assertSafeIri(asset.headSubject)}> ?shareId . ` + - `?op ?shareId ; ` + - ` <${assertSafeIri(asset.kaUal)}> . } }`, - { - priority: 'background', - source: 'agent.swmRecovery.replaceMetaForGraphAssets.findOperations', - }, - ); - const operationSubjects = new Set([asset.operationSubject]); - if (linkedOperations.type === 'bindings') { - for (const row of linkedOperations.bindings) { - const operation = row['op']; - if (operation) operationSubjects.add(operation); - } - } - await dependencies.store.deleteByPattern( - { graph: asset.metaGraph, subject: asset.headSubject }, - { - priority: 'background', - source: 'agent.swmRecovery.replaceMetaForGraphAssets.deleteHead', - }, + await withKeyedLocks( + dependencies.writeLocks, + [swmKaWriteLockKey(contextGraphId, asset.subGraphName, asset.kaUal)], + () => replaceGraphScopedSwmHeadMetadata({ + store: dependencies.store, + contextGraphId, + descriptor: asset, + sourcePrefix: 'agent.swmRecovery.replaceMetaForGraphAssets', + }), ); - for (const operationSubject of operationSubjects) { - await dependencies.store.deleteByPattern( - { graph: asset.metaGraph, subject: operationSubject }, - { - priority: 'background', - source: 'agent.swmRecovery.replaceMetaForGraphAssets.deleteOperation', - }, - ); - } } }, ensureContextGraph: async (cgId) => { diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index c8f9402c22..9b9e439609 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -98,6 +98,8 @@ import { type VerifiedGraphScopedFinalizationEvidence, } from './finalization-graph-envelope.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; +import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from './dkg-agent-constants.js'; +export { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from './dkg-agent-constants.js'; /** * Predicate for the durable per-root keep-root-copy signal the publisher @@ -141,7 +143,6 @@ export const KEEP_ROOT_COPY_PREDICATE = `${DKG_NS}keepRootCopyOnLabel`; */ export const SWM_SNAPSHOT_MERKLE_ROOT_PREDICATE = `${DKG_NS}snapshotMerkleRoot`; export const SWM_SNAPSHOT_CONTENT_DIGEST_PREDICATE = `${DKG_NS}snapshotContentDigest`; -export const FINALIZED_SWM_CLEANUP_ROOT_PREDICATE = `${DKG_NS}finalizedSwmCleanupRoot`; /** * Resolves a local context-graph id (the topic/CG name used in gossip) to diff --git a/packages/agent/src/sync/graph-scoped-swm-recovery.ts b/packages/agent/src/sync/graph-scoped-swm-recovery.ts index 2434ac4a8c..374cf53f74 100644 --- a/packages/agent/src/sync/graph-scoped-swm-recovery.ts +++ b/packages/agent/src/sync/graph-scoped-swm-recovery.ts @@ -44,9 +44,13 @@ export interface GraphScopedSwmRecoveryDescriptor { readonly shareOperationId: string; readonly publicQuadsDigest: string; readonly publicQuadsCount: number; + readonly privateMerkleRoot?: string; + readonly privateTripleCount: number; readonly publicSnapshotRef?: string; readonly publicSnapshotGraph?: string; readonly publisherPeerId: string; + readonly accessPolicy?: 'public' | 'ownerOnly' | 'allowList'; + readonly allowedPeers: readonly string[]; readonly subGraphName?: string; /** Only the active head and its referenced operation, for snapshot fetch. */ readonly metadataQuads: readonly Quad[]; @@ -222,6 +226,15 @@ export function parseGraphScopedSwmRecoveryDescriptors(params: { if (!publisherPeerId) { throw new Error(`Graph-scoped SWM operation ${operationSubject} has an empty publisherPeerId`); } + const accessPolicy = optionalLiteral(operationRows, ACCESS_POLICY, 'accessPolicy')?.trim() as + | 'public' + | 'ownerOnly' + | 'allowList' + | undefined; + const allowedPeers = distinctObjects(operationRows, ALLOWED_PEER) + .map(stripLiteral) + .filter(Boolean) + .sort(); descriptors.push({ metaGraph, headSubject, @@ -232,9 +245,13 @@ export function parseGraphScopedSwmRecoveryDescriptors(params: { shareOperationId, publicQuadsDigest, publicQuadsCount, + ...(privateRoot ? { privateMerkleRoot: stripLiteral(privateRoot).toLowerCase() } : {}), + privateTripleCount, ...(publicSnapshotRef ? { publicSnapshotRef } : {}), ...(publicSnapshotGraph ? { publicSnapshotGraph } : {}), publisherPeerId, + ...(accessPolicy ? { accessPolicy } : {}), + allowedPeers, ...(subGraphName ? { subGraphName } : {}), metadataQuads: [...headRows, ...operationRows], }); diff --git a/packages/agent/src/sync/requester/shared-memory-sync.ts b/packages/agent/src/sync/requester/shared-memory-sync.ts index 9e0edf6d0d..191161845e 100644 --- a/packages/agent/src/sync/requester/shared-memory-sync.ts +++ b/packages/agent/src/sync/requester/shared-memory-sync.ts @@ -296,6 +296,9 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro let materializationFailures = 0; let materializedQuads = 0; const materializedKeys = new Set(); + const replacedGraphScopedMetaKeys = new Set(); + const quadKey = (quad: Quad): string => + `${quad.graph}\u0000${quad.subject}\u0000${quad.predicate}\u0000${quad.object}`; const materializeReadySnapshot = async (snapshotRef: string): Promise => { const descriptors = snapshotDescriptorsByRef.get(snapshotRef); if (!descriptors?.length || !snapshotMaterializer || !publicSnapshotStore) return; @@ -327,6 +330,9 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro storedHead.version !== null && storedVersionOutranksDescriptor(storedHead.version, descriptor.assertionVersion) ) { + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); + } materializedKeys.add(graphKey); logDebug(ctx, `SWM sync for "${pid}": snapshot ${snapshotRef} superseded by ` + `stored version ${storedHead.version} (descriptor ${descriptor.assertionVersion}); skipping`); @@ -338,16 +344,20 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // digest is an OLDER version of the same size and must be // replaced, not skipped. if (await snapshotMaterializer.isGraphAssetMaterialized(descriptor)) { - if (storedHead.needsRepair) { + if (storedHead.version === null || storedHead.needsRepair) { // Content is already this descriptor's, but the head - // subject still carries union-insert residue (several - // version/operation rows) — e.g. a prior round replaced - // the graph and then failed before finishing the metadata - // swap. Collapse the head now; the fresh verified meta for - // this descriptor is re-inserted after the snapshot phase, - // exactly like the replace path below. + // is absent or still carries union-insert residue (several + // version/operation rows) — e.g. a prior round replaced the + // graph and then failed before finishing the metadata swap. + // Install the verified head while the writer lock is held. await snapshotMaterializer.replaceHeadMetadata(pid, descriptor); } + // Never append graph-scoped metadata again after releasing + // this lock. For a clean head it is redundant; for a newer + // local lifecycle it would recreate stale union residue. + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); + } materializedKeys.add(graphKey); return; } @@ -361,11 +371,15 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // Graph first, THEN the head swap — a crash between the two // leaves content newer than the head, which the next round // repairs (digest matches → head collapsed above). The swap - // deletes the old head + its operations so the append-style - // `storeInsert(processed.verifiedMeta)` below lands on a clean - // subject instead of stacking a second version onto it - // (LIMIT-1 head readers would otherwise see an arbitrary mix). + // deletes the old head + its operations, installs the verified + // replacement under this same writer lock, and excludes those + // rows from the append-style metadata insert below. Otherwise + // a live write could land between the swap and the append and + // leave an arbitrary multi-version head. await snapshotMaterializer.replaceHeadMetadata(pid, descriptor); + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); + } materializedKeys.add(graphKey); materializedGraphs += 1; materializedQuads += asset.quads.length; @@ -460,8 +474,13 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro summary.insertedTriples += validWsQuads.length; summary.insertedDataTriples += validWsQuads.length; } + const remainingVerifiedMeta = processed.verifiedMeta.filter( + (quad) => !replacedGraphScopedMetaKeys.has(quadKey(quad)), + ); + if (remainingVerifiedMeta.length > 0) { + await storeInsert(remainingVerifiedMeta); + } if (processed.verifiedMeta.length > 0) { - await storeInsert(processed.verifiedMeta); summary.insertedTriples += processed.verifiedMeta.length; summary.insertedMetaTriples += processed.verifiedMeta.length; } diff --git a/packages/agent/src/sync/requester/swm-recovery.ts b/packages/agent/src/sync/requester/swm-recovery.ts index 4b38bfe644..25389dc495 100644 --- a/packages/agent/src/sync/requester/swm-recovery.ts +++ b/packages/agent/src/sync/requester/swm-recovery.ts @@ -97,7 +97,10 @@ export interface RecoverContextGraphSwmDeps { roots: readonly { readonly entity: string }[], metaGraphs: readonly string[], ) => Promise; - /** Replace the active head/operation rows for each exact graph asset. */ + /** + * Replace and insert the active head/operation rows for each exact graph + * asset while holding the production per-KA writer lock. + */ readonly replaceMetaForGraphAssets?: ( assets: readonly GraphScopedSwmRecoveryDescriptor[], ) => Promise; @@ -343,8 +346,9 @@ export async function recoverContextGraphSwm( // A crash between the two therefore retries idempotently; it can never // advertise a head whose graph was only partially transferred. await deps.store.replaceGraph(asset.assertionGraph, [...asset.quads]); - await deps.replaceMetaForGraphAssets?.([descriptor]); - if (verifiedAssetMeta.length > 0) { + if (deps.replaceMetaForGraphAssets) { + await deps.replaceMetaForGraphAssets([descriptor]); + } else if (verifiedAssetMeta.length > 0) { await deps.store.insert([...verifiedAssetMeta]); } incrementallyReadyGraphs.add(graphKey); @@ -487,8 +491,14 @@ export async function recoverContextGraphSwm( if (graphScopedDescriptors.length > 0) { await deps.replaceMetaForGraphAssets?.(graphScopedDescriptors); } - if (processed.verifiedMeta.length > 0) { - await deps.store.insert([...processed.verifiedMeta]); + const replacedGraphScopedMetaKeys = deps.replaceMetaForGraphAssets + ? new Set(graphScopedDescriptors.flatMap((descriptor) => descriptor.metadataQuads).map(quadKey)) + : new Set(); + const remainingVerifiedMeta = processed.verifiedMeta.filter( + (quad) => !replacedGraphScopedMetaKeys.has(quadKey(quad)), + ); + if (remainingVerifiedMeta.length > 0) { + await deps.store.insert([...remainingVerifiedMeta]); } // R2 — hydrate the Rule-4 ownership cache for the recovered roots (parity with diff --git a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts index 08874827fd..7277fb8f50 100644 --- a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts +++ b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts @@ -15,12 +15,15 @@ */ import { assertSafeIri } from '@origintrail-official/dkg-core'; import { + resolveKnowledgeAssetWorkspaceHead, + sameKnowledgeAssetWorkspaceHead, swmKaWriteLockKey, withKeyedLocks, workspacePublicQuadsDigest, } from '@origintrail-official/dkg-publisher'; -import type { Quad, TripleStore } from '@origintrail-official/dkg-storage'; +import { GraphManager, type Quad, type TripleStore } from '@origintrail-official/dkg-storage'; import type { GraphScopedSwmRecoveryDescriptor } from '../graph-scoped-swm-recovery.js'; +import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from '../../dkg-agent-constants.js'; const DKG = 'http://dkg.io/ontology/'; @@ -91,13 +94,13 @@ export interface SharedMemorySnapshotMaterializer { */ replaceGraph(graphUri: string, quads: Quad[]): Promise; /** - * Delete the KA's head rows and every share-operation subject its head - * references (including the descriptor's own, which the caller re-inserts - * from fresh verified metadata). This is the catch-up lane's counterpart of - * gossip's delete-then-insert (`storeKnowledgeAssetWorkspaceHead`) and the - * private recovery lane's `replaceMetaForGraphAssets`: without it the - * append-style meta insert stacks old and new head rows on one subject and - * the durable current head becomes ambiguous. + * Replace the KA's head rows and every share-operation subject its head + * references from fresh verified metadata. This is the catch-up lane's + * counterpart of gossip's delete-then-insert + * (`storeKnowledgeAssetWorkspaceHead`) and the private recovery lane's + * `replaceMetaForGraphAssets`: without it the append-style meta insert stacks + * old and new head rows on one subject and the durable current head becomes + * ambiguous. */ replaceHeadMetadata( contextGraphId: string, @@ -105,6 +108,152 @@ export interface SharedMemorySnapshotMaterializer { ): Promise; } +/** + * Replace one graph-scoped SWM lifecycle without losing a local deferred + * finalization token for the exact same lifecycle. + * + * Finalization markers are deliberately local-only and responders filter them + * from synchronized metadata. A blind head/operation replacement therefore + * erased the only durable evidence that the retained SWM graph was eligible + * for idle cleanup. Preserve both marker rows only when the complete local + * workspace head still equals the verified incoming descriptor; any mismatch + * means a newer or otherwise different lifecycle and fails closed by dropping + * the old markers. + * + * The verified replacement metadata and any retained cleanup markers are + * inserted in the same store call after the old lifecycle is removed. Callers + * MUST hold the canonical per-KA SWM writer lock across this entire operation. + */ +export async function replaceGraphScopedSwmHeadMetadata(params: { + store: TripleStore; + contextGraphId: string; + descriptor: GraphScopedSwmRecoveryDescriptor; + sourcePrefix: string; +}): Promise { + const { store, contextGraphId, descriptor, sourcePrefix } = params; + const preservedMarkers = await readExactFinalizedCleanupMarkers({ + store, + contextGraphId, + descriptor, + sourcePrefix, + }); + + // Collect every share operation the head currently references — via the + // BOUND head subject, then per-candidate bound-subject ASKs. The kaUal guard + // prevents a corrupt cross-KA reference from deleting another KA's metadata. + const shareIds = await store.query( + `SELECT DISTINCT ?op WHERE { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(descriptor.headSubject)}> <${DKG}shareOperationId> ?op } }`, + { priority: 'background', source: `${sourcePrefix}.findOperations` }, + ); + const operationSubjects = new Set([descriptor.operationSubject]); + if (shareIds.type === 'bindings') { + for (const row of shareIds.bindings) { + const shareId = literalValue(row?.['op']); + if (!shareId) continue; + const candidate = `urn:dkg:share:${contextGraphId}:${shareId}`; + if (operationSubjects.has(candidate)) continue; + const ownedByThisKa = await store.query( + `ASK { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(candidate)}> <${DKG}kaUal> <${assertSafeIri(descriptor.kaUal)}> } }`, + { priority: 'background', source: `${sourcePrefix}.checkOperation` }, + ); + if (ownedByThisKa.type === 'boolean' && ownedByThisKa.value) { + operationSubjects.add(candidate); + } + } + } + + await store.deleteByPattern( + { graph: descriptor.metaGraph, subject: descriptor.headSubject }, + { priority: 'background', source: `${sourcePrefix}.deleteHead` }, + ); + for (const operationSubject of operationSubjects) { + await store.deleteByPattern( + { graph: descriptor.metaGraph, subject: operationSubject }, + { priority: 'background', source: `${sourcePrefix}.deleteOperation` }, + ); + } + const replacementQuads = [ + ...descriptor.metadataQuads, + ...preservedMarkers, + ]; + if (replacementQuads.length > 0) { + await store.insert(replacementQuads, { + priority: 'background', + source: `${sourcePrefix}.insertReplacementMetadata`, + }); + } +} + +async function readExactFinalizedCleanupMarkers(params: { + store: TripleStore; + contextGraphId: string; + descriptor: GraphScopedSwmRecoveryDescriptor; + sourcePrefix: string; +}): Promise { + const { store, contextGraphId, descriptor, sourcePrefix } = params; + try { + const currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager: new GraphManager(store), + contextGraphId, + kaUal: descriptor.kaUal, + subGraphName: descriptor.subGraphName, + queryOptions: { + priority: 'background', + source: `${sourcePrefix}.readFinalizedCleanupHead`, + }, + }); + if (!currentHead || !sameKnowledgeAssetWorkspaceHead(currentHead, { + kaUal: descriptor.kaUal, + assertionVersion: descriptor.assertionVersion, + assertionGraph: descriptor.assertionGraph, + publicQuadsDigest: descriptor.publicQuadsDigest, + publicTripleCount: descriptor.publicQuadsCount, + privateMerkleRoot: descriptor.privateMerkleRoot, + privateTripleCount: descriptor.privateTripleCount, + shareOperationId: descriptor.shareOperationId, + publisherPeerId: descriptor.publisherPeerId, + accessPolicy: descriptor.accessPolicy, + allowedPeers: [...descriptor.allowedPeers], + })) { + return []; + } + } catch { + // Corrupt or incomplete local metadata must never carry a destructive + // cleanup token into a verified replacement. + return []; + } + + const markerResult = await store.query( + `CONSTRUCT { ?subject <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } WHERE { ` + + `GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `VALUES ?subject { <${assertSafeIri(descriptor.headSubject)}> ` + + `<${assertSafeIri(descriptor.operationSubject)}> } ` + + `?subject <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + { + priority: 'background', + source: `${sourcePrefix}.readFinalizedCleanupMarkers`, + }, + ); + if (markerResult.type !== 'quads') return []; + const markers = markerResult.quads.map((quad) => ({ + ...quad, + graph: descriptor.metaGraph, + })); + const headMarkers = markers.filter((quad) => quad.subject === descriptor.headSubject); + const operationMarkers = markers.filter((quad) => quad.subject === descriptor.operationSubject); + if ( + headMarkers.length !== 1 + || operationMarkers.length !== 1 + || headMarkers[0]!.object !== operationMarkers[0]!.object + ) { + return []; + } + return [headMarkers[0]!, operationMarkers[0]!]; +} + /** * Build the production materializer over the agent's own store, lock map and * list-cache invalidation hook. @@ -193,43 +342,13 @@ export function createSharedMemorySnapshotMaterializer(deps: { }, replaceHeadMetadata: async (contextGraphId, descriptor) => { - // Collect every share operation the head currently references — via the - // BOUND head subject, then per-candidate bound-subject ASKs, so no query - // scans the meta bucket. The kaUal guard mirrors the recovery lane's - // `replaceMetaForGraphAssets` join: a head row pointing at another KA's - // operation must not delete that KA's metadata. - const shareIds = await deps.store.query( - `SELECT DISTINCT ?op WHERE { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` - + `<${assertSafeIri(descriptor.headSubject)}> <${DKG}shareOperationId> ?op } }`, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.findOperations' }, - ); - const operationSubjects = new Set([descriptor.operationSubject]); - if (shareIds.type === 'bindings') { - for (const row of shareIds.bindings) { - const shareId = literalValue(row?.['op']); - if (!shareId) continue; - const candidate = `urn:dkg:share:${contextGraphId}:${shareId}`; - if (operationSubjects.has(candidate)) continue; - const ownedByThisKa = await deps.store.query( - `ASK { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` - + `<${assertSafeIri(candidate)}> <${DKG}kaUal> <${assertSafeIri(descriptor.kaUal)}> } }`, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.checkOperation' }, - ); - if (ownedByThisKa.type === 'boolean' && ownedByThisKa.value) { - operationSubjects.add(candidate); - } - } - } - await deps.store.deleteByPattern( - { graph: descriptor.metaGraph, subject: descriptor.headSubject }, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.deleteHead' }, - ); - for (const operationSubject of operationSubjects) { - await deps.store.deleteByPattern( - { graph: descriptor.metaGraph, subject: operationSubject }, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.deleteOperation' }, - ); - } + await replaceGraphScopedSwmHeadMetadata({ + store: deps.store, + contextGraphId, + descriptor, + sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + }); + deps.invalidateListContextGraphsCache(); }, }; } diff --git a/packages/agent/test/swm-public-snapshot-materialization.test.ts b/packages/agent/test/swm-public-snapshot-materialization.test.ts index 276e279440..d5f2dcf13e 100644 --- a/packages/agent/test/swm-public-snapshot-materialization.test.ts +++ b/packages/agent/test/swm-public-snapshot-materialization.test.ts @@ -16,8 +16,8 @@ * 4. an already-materialized asset is left alone * 5. a failed replace keeps the phase incomplete and WITHHOLDS the meta * insert, so a marker can never certify a graph that was not written - * 6. after a replace, the stale head metadata is swapped out BEFORE the - * fresh verified meta is appended (graph → head swap → meta ordering) + * 6. after a graph replace, stale head metadata is replaced with the fresh + * verified rows inside the same per-KA lock (graph → locked metadata swap) * 7. a KA under a REGISTERED subgraph is materialized too — dropping the * subgraph admission pass-through must fail these tests * 8. a node with NO cached snapshot fetches it over the network @@ -200,6 +200,8 @@ function harness(overrides: HarnessOverrides = {}) { replaceHeadMetadata: async (contextGraphId, descriptor) => { events.push('head-swapped'); headSwaps.push({ contextGraphId, headSubject: descriptor.headSubject }); + inserted.push([...descriptor.metadataQuads]); + events.push('meta-replaced'); }, }, publicSnapshotStore: snapshotStore, @@ -229,18 +231,16 @@ describe('public SWM snapshot materialization', () => { expect(h.inserted.some((batch) => batch.some((q) => q.graph === WS_META))).toBe(true); }); - it('swaps the stale head metadata after the replace and BEFORE the meta append', async () => { - // The meta insert below is append/union-style: without the head swap the - // old and new version rows stack on one subject and LIMIT-1 readers - // (`resolveKnowledgeAssetWorkspaceHead`) can return either — the stale - // head bug. Ordering matters both ways: graph before swap (a crash leaves - // repairable content, never a head without content), swap before append - // (the fresh rows land on a clean subject). + it('replaces stale head metadata inside the locked swap after replacing the graph', async () => { + // The verified graph-scoped metadata must land in the same locked step as + // the delete. Appending it later would let a live write slip between the + // two operations and recreate a multi-version head. const h = harness({ storedHead: () => ({ version: '1', needsRepair: false }), contentPresent: () => false }); await h.run(); expect(h.events.indexOf('replaced')).toBeGreaterThan(-1); expect(h.events.indexOf('head-swapped')).toBeGreaterThan(h.events.indexOf('replaced')); - expect(h.events.indexOf('meta-inserted')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events.indexOf('meta-replaced')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events).not.toContain('meta-inserted'); expect(h.headSwaps).toEqual([{ contextGraphId: CG, headSubject: `${UAL}#dkg-swm-head` }]); }); @@ -285,6 +285,7 @@ describe('public SWM snapshot materialization', () => { expect(h.events.indexOf('version-read')).toBeGreaterThan(h.events.indexOf('gossip-committed')); expect(h.events).not.toContain('replaced'); expect(h.events).not.toContain('head-swapped'); + expect(h.inserted.every((batch) => batch.every((q) => q.graph !== WS_META))).toBe(true); expect(summary.failedPhases).toBe(0); }); @@ -316,7 +317,8 @@ describe('public SWM snapshot materialization', () => { const summary = await h.run(); expect(h.events).not.toContain('replaced'); expect(h.events).toContain('head-swapped'); - expect(h.events.indexOf('meta-inserted')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events.indexOf('meta-replaced')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events).not.toContain('meta-inserted'); expect(summary.failedPhases).toBe(0); }); diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index 381a1a432f..393d90a5fc 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -41,12 +41,14 @@ import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; import { runSharedMemorySync } from '../src/sync/requester/shared-memory-sync.js'; import type { SyncPageResult } from '../src/sync/requester/page-fetch.js'; +import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from '../src/finalization-handler.js'; const CG = 'ws00-materializer-real-store'; const WS_META = contextGraphWorkspaceMetaGraphUri(CG); const DKG = 'http://dkg.io/ontology/'; const XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer'; const UAL = 'did:dkg:hardhat:31337/0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/9'; +const FINALIZED_ROOT = `"0x${'11'.repeat(32)}"`; const ctx: OperationContext = { operationId: 'test', operationName: 'sync' } as never; class MemorySnapshotStore implements WorkspacePublicSnapshotStore { @@ -199,7 +201,7 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', }); describe('replaceHeadMetadata', () => { - it('deletes the head and every referenced operation, sparing unrelated subjects', async () => { + it('replaces the head and every referenced operation, sparing unrelated subjects', async () => { const store = new OxigraphStore(); await store.insert([...v1.meta]); await store.insert([...v2.meta]); @@ -214,9 +216,11 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', await materializer.replaceHeadMetadata(CG, descriptorFor(v2)); - expect(await distinctObjects(store, WS_META, v2.headSubject, `${DKG}assertionVersion`)).toEqual([]); + expect(await distinctObjects(store, WS_META, v2.headSubject, `${DKG}assertionVersion`)) + .toEqual([`"2"^^<${XSD_INTEGER}>`]); expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)).toEqual([]); - expect(await distinctObjects(store, WS_META, v2.operationSubject, `${DKG}shareOperationId`)).toEqual([]); + expect(await distinctObjects(store, WS_META, v2.operationSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v2.operationId}"`]); expect(await distinctObjects(store, WS_META, unrelated.subject, `${DKG}shareOperationId`)).toEqual(['"unrelated"']); }); @@ -237,10 +241,90 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', await materializer.replaceHeadMetadata(CG, descriptorFor(v1)); - expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)).toEqual([]); - expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)).toEqual([]); + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v1.operationId}"`]); + expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v1.operationId}"`]); expect(await distinctObjects(store, WS_META, foreignOp, `${DKG}shareOperationId`)).toEqual(['"foreign-op"']); }); + + it('preserves the local finalized-cleanup token when synchronized metadata is the exact same lifecycle', async () => { + const store = new OxigraphStore(); + await store.insert([ + ...v1.meta, + { + subject: v1.headSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + ]); + const { materializer } = materializerFor(store); + + await materializer.withKaWriteLock(CG, undefined, UAL, () => + materializer.replaceHeadMetadata(CG, descriptorFor(v1))); + + expect(await distinctObjects( + store, + WS_META, + v1.headSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([FINALIZED_ROOT]); + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([FINALIZED_ROOT]); + }); + + it('drops the old finalized-cleanup token when synchronized metadata is a newer lifecycle', async () => { + const store = new OxigraphStore(); + await store.insert([ + ...v1.meta, + { + subject: v1.headSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + ]); + const { materializer } = materializerFor(store); + + await materializer.withKaWriteLock(CG, undefined, UAL, () => + materializer.replaceHeadMetadata(CG, descriptorFor(v2))); + + expect(await distinctObjects( + store, + WS_META, + v2.headSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + expect(await distinctObjects( + store, + WS_META, + v2.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + }); }); it('replaceGraph writes atomically and invalidates the list cache', async () => { diff --git a/packages/agent/test/sync-requester-priority.test.ts b/packages/agent/test/sync-requester-priority.test.ts index dcb760befb..acb691fbf4 100644 --- a/packages/agent/test/sync-requester-priority.test.ts +++ b/packages/agent/test/sync-requester-priority.test.ts @@ -210,6 +210,7 @@ describe('requester per-CG priority admission', () => { const admissions: string[] = []; const warnings: string[] = []; const contextGraphIds = ['first', 'second', 'third']; + let cleanupOptions: unknown; const agent = { config: { syncContextGraphPriorities: {} }, store: {}, @@ -251,6 +252,10 @@ describe('requester per-CG priority admission', () => { }, syncCheckpoints: new Map(), workspaceOwnedEntities: new Map(), + cleanupExpiredSharedMemory: async (options: unknown) => { + cleanupOptions = options; + return 0; + }, log: { info: noop, warn: (_ctx: unknown, message: string) => warnings.push(message), @@ -275,6 +280,11 @@ describe('requester per-CG priority admission', () => { expect(summary.deferredBackpressure).toBe(1); expect(summary.failedPeers).toBe(0); expect(summary.backoffWorthyFailures).toBe(0); + expect(cleanupOptions).toEqual({ + finalizedOnly: true, + contextGraphIds, + finalizedCleanupBudget: 64, + }); }); it('counts several failed Context Graphs from one remote as one failed peer', async () => { From ee1a53390c794b50d86374b3668d67e441e11fbe Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 16:35:18 +0200 Subject: [PATCH 04/48] fix(agent): drain finalized SWM after catchup --- packages/agent/src/dkg-agent-lifecycle.ts | 33 +++---- packages/agent/src/finalization-handler.ts | 16 +++- packages/agent/test/agent.part-16.test.ts | 85 ++++++++++++------- .../ka-graph-finalization-handler.test.ts | 30 +++++++ .../test/sync-requester-priority.test.ts | 10 --- 5 files changed, 112 insertions(+), 62 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 97d10f3aac..772e94a227 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -5447,22 +5447,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { `Deferring ${item.lane} at CG ${item.contextGraphId} due to local backpressure: ${error.message}`, ), }); - if ( - work.length > 0 - && summary.failedPhases === 0 - && typeof this.cleanupExpiredSharedMemory === 'function' - ) { - // A completed catch-up is the first reliable low-pressure boundary - // after a large synchronized write. Drain finalized duplicates here, - // before reporting the job complete, in small background-priority - // batches that stop as soon as normal work appears. The periodic - // maintenance timer remains the restart/failure backstop. - await this.cleanupExpiredSharedMemory({ - finalizedOnly: true, - contextGraphIds: work.map((item) => item.contextGraphId), - finalizedCleanupBudget: 64, - }); - } return summary; }; @@ -6055,6 +6039,21 @@ export class LifecycleSyncMethods extends DKGAgentBase { verifiedPrivateOnlyResponses: cleanDurablePrivateOnlyCompletions, }); } + if (includeSharedMemory && typeof this.cleanupExpiredSharedMemory === 'function') { + // Every selected peer has settled, so this is the first single, + // deterministic cleanup boundary for the whole catch-up job. The + // cleanup's store operations remain background-priority and its + // per-KA lock/exact VM+SWM+marker re-checks preserve a concurrent newer + // SWM lifecycle. Background sync traffic must not starve this drain: + // only foreground/ACK pressure defers it, with the periodic timer as the + // restart/failure backstop. + await this.cleanupExpiredSharedMemory({ + finalizedOnly: true, + contextGraphIds: [contextGraphId], + finalizedCleanupBudget: 64, + allowDuringBackgroundPressure: true, + }); + } return { connectedPeers: stats?.totalPeers ?? peers.length, @@ -7594,6 +7593,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { finalizedOnly?: boolean; contextGraphIds?: readonly string[]; finalizedCleanupBudget?: number; + allowDuringBackgroundPressure?: boolean; }): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; const ctx = createOperationContext('share'); @@ -7637,6 +7637,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { contextGraphId: pid, swmMetaGraph: wsMetaGraph, maxCandidates: batchSize, + allowDuringBackgroundPressure: options?.allowDuringBackgroundPressure, }); finalizedCleanupBudget -= cleaned; if (cleaned < batchSize) break; diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index 9b9e439609..75bcb573ce 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -1608,7 +1608,9 @@ export class FinalizationHandler { /** * Drain a bounded number of durable finalized-SWM markers only while the - * store scheduler reports no queued or in-flight work. + * store scheduler reports no queued or in-flight work. The deterministic + * post-catch-up boundary may explicitly tolerate unrelated background work; + * periodic maintenance remains fully idle-only. * * The marker survives restart, while replacing the SWM head for a newer * assertion removes it automatically. All maintenance queries run in the @@ -1619,6 +1621,7 @@ export class FinalizationHandler { contextGraphId: string; swmMetaGraph: string; maxCandidates?: number; + allowDuringBackgroundPressure?: boolean; }): Promise { if (!this.writeLocks) return 0; const pressure = this.store.getPressureSnapshot?.(); @@ -1626,11 +1629,11 @@ export class FinalizationHandler { pressure.ackInflight > 0 || (pressure.healthInflight ?? 0) > 0 || pressure.normalInflight > 0 - || pressure.backgroundInflight > 0 + || (!input.allowDuringBackgroundPressure && pressure.backgroundInflight > 0) || pressure.ackQueued > 0 || (pressure.healthQueued ?? 0) > 0 || pressure.normalQueued > 0 - || pressure.backgroundQueued > 0 + || (!input.allowDuringBackgroundPressure && pressure.backgroundQueued > 0) )) { return 0; } @@ -1661,9 +1664,14 @@ export class FinalizationHandler { for (const row of result.bindings) { const currentPressure = this.store.getPressureSnapshot?.(); if (currentPressure && ( - currentPressure.ackQueued > 0 + currentPressure.ackInflight > 0 + || (currentPressure.healthInflight ?? 0) > 0 + || currentPressure.normalInflight > 0 + || (!input.allowDuringBackgroundPressure && currentPressure.backgroundInflight > 0) + || currentPressure.ackQueued > 0 || (currentPressure.healthQueued ?? 0) > 0 || currentPressure.normalQueued > 0 + || (!input.allowDuringBackgroundPressure && currentPressure.backgroundQueued > 0) )) { break; } diff --git a/packages/agent/test/agent.part-16.test.ts b/packages/agent/test/agent.part-16.test.ts index 2587ee8993..7fcc80d54c 100644 --- a/packages/agent/test/agent.part-16.test.ts +++ b/packages/agent/test/agent.part-16.test.ts @@ -125,40 +125,52 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => // `insertedTriples` count exposed by `syncFromPeer` / `syncSharedMemoryFromPeer`. // Replace those with recorders so we can assert both the call shape and // the reported totals without spinning up a remote peer. - const syncFromPeerDetailed = recorder(async () => ({ - insertedTriples: 5, - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 5, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 0, - metaOnlyResponses: 0, - dataRejectedMissingMeta: 0, - rejectedKcs: 0, - failedPeers: 0, - })); + const lifecycleOrder: string[] = []; + const syncFromPeerDetailed = recorder(async () => { + lifecycleOrder.push('durable'); + return { + insertedTriples: 5, + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 5, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 0, + metaOnlyResponses: 0, + dataRejectedMissingMeta: 0, + rejectedKcs: 0, + failedPeers: 0, + }; + }); (agent as any).syncFromPeerDetailed = syncFromPeerDetailed; - const syncSharedMemoryFromPeerDetailed = recorder(async () => ({ - insertedTriples: 2, - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 2, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 0, - droppedDataTriples: 0, - failedPeers: 0, - })); + const syncSharedMemoryFromPeerDetailed = recorder(async () => { + lifecycleOrder.push('shared-memory'); + return { + insertedTriples: 2, + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 2, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 0, + droppedDataTriples: 0, + failedPeers: 0, + }; + }); (agent as any).syncSharedMemoryFromPeerDetailed = syncSharedMemoryFromPeerDetailed; + const cleanupExpiredSharedMemory = recorder(async () => { + lifecycleOrder.push('cleanup'); + return 0; + }); + (agent as any).cleanupExpiredSharedMemory = cleanupExpiredSharedMemory; const result = await agent.syncContextGraphFromConnectedPeers('runtime-contextGraph', { includeSharedMemory: true, @@ -186,6 +198,15 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => expect(result.diagnostics.noProtocolPeers).toBe(0); expect(result.diagnostics.durable.failedPeers).toBe(0); expect(result.diagnostics.sharedMemory.failedPeers).toBe(0); + expect(cleanupExpiredSharedMemory.calls).toEqual([[ + { + finalizedOnly: true, + contextGraphIds: ['runtime-contextGraph'], + finalizedCleanupBudget: 64, + allowDuringBackgroundPressure: true, + }, + ]]); + expect(lifecycleOrder).toEqual(['durable', 'shared-memory', 'cleanup']); expect(agent.getSubscribedContextGraphs().get('runtime-contextGraph')).toMatchObject({ synced: true, sharedMemorySynced: true, diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 4f2e31ff0a..5f393a1db2 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2220,6 +2220,36 @@ describe('graph-scoped finalization handler', () => { }); }); + it('does not starve finalized cleanup behind unrelated background sync pressure', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: 0, + backgroundInflight: 2, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 4, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + + expect(await drainFinalizedSwm()).toBe(0); + expect(await handler.cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + allowDuringBackgroundPressure: true, + })).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + }); + it('repairs VM from the immutable operation snapshot after deferred SWM cleanup', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); diff --git a/packages/agent/test/sync-requester-priority.test.ts b/packages/agent/test/sync-requester-priority.test.ts index acb691fbf4..dcb760befb 100644 --- a/packages/agent/test/sync-requester-priority.test.ts +++ b/packages/agent/test/sync-requester-priority.test.ts @@ -210,7 +210,6 @@ describe('requester per-CG priority admission', () => { const admissions: string[] = []; const warnings: string[] = []; const contextGraphIds = ['first', 'second', 'third']; - let cleanupOptions: unknown; const agent = { config: { syncContextGraphPriorities: {} }, store: {}, @@ -252,10 +251,6 @@ describe('requester per-CG priority admission', () => { }, syncCheckpoints: new Map(), workspaceOwnedEntities: new Map(), - cleanupExpiredSharedMemory: async (options: unknown) => { - cleanupOptions = options; - return 0; - }, log: { info: noop, warn: (_ctx: unknown, message: string) => warnings.push(message), @@ -280,11 +275,6 @@ describe('requester per-CG priority admission', () => { expect(summary.deferredBackpressure).toBe(1); expect(summary.failedPeers).toBe(0); expect(summary.backoffWorthyFailures).toBe(0); - expect(cleanupOptions).toEqual({ - finalizedOnly: true, - contextGraphIds, - finalizedCleanupBudget: 64, - }); }); it('counts several failed Context Graphs from one remote as one failed peer', async () => { From 1d9fcfc1c6f8ae1fef3bba8604a6191b97d1c668 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 17:31:15 +0200 Subject: [PATCH 05/48] fix(agent): queue finalized SWM cleanup after catchup --- packages/agent/src/dkg-agent-lifecycle.ts | 15 ++++++++------- packages/agent/src/finalization-handler.ts | 16 ++++++++-------- packages/agent/test/agent.part-16.test.ts | 2 +- .../test/ka-graph-finalization-handler.test.ts | 16 ++++++++-------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 772e94a227..869499fb89 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -6044,14 +6044,15 @@ export class LifecycleSyncMethods extends DKGAgentBase { // deterministic cleanup boundary for the whole catch-up job. The // cleanup's store operations remain background-priority and its // per-KA lock/exact VM+SWM+marker re-checks preserve a concurrent newer - // SWM lifecycle. Background sync traffic must not starve this drain: - // only foreground/ACK pressure defers it, with the periodic timer as the - // restart/failure backstop. + // SWM lifecycle. Queue behind active store work instead of sampling + // pressure and abandoning the drain: the store scheduler keeps + // foreground/ACK priority and guarantees background progress. The + // periodic timer remains the restart/failure backstop. await this.cleanupExpiredSharedMemory({ finalizedOnly: true, contextGraphIds: [contextGraphId], finalizedCleanupBudget: 64, - allowDuringBackgroundPressure: true, + queueBehindActiveWork: true, }); } @@ -7593,7 +7594,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { finalizedOnly?: boolean; contextGraphIds?: readonly string[]; finalizedCleanupBudget?: number; - allowDuringBackgroundPressure?: boolean; + queueBehindActiveWork?: boolean; }): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; const ctx = createOperationContext('share'); @@ -7637,12 +7638,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { contextGraphId: pid, swmMetaGraph: wsMetaGraph, maxCandidates: batchSize, - allowDuringBackgroundPressure: options?.allowDuringBackgroundPressure, + queueBehindActiveWork: options?.queueBehindActiveWork, }); finalizedCleanupBudget -= cleaned; if (cleaned < batchSize) break; // Yield between bounded batches so queued foreground work can - // arrive and trip the handler's pressure gate. + // be admitted ahead of the next background-priority batch. await new Promise((resolve) => setImmediate(resolve)); } } catch (error) { diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index 75bcb573ce..ef7fe62522 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -1609,7 +1609,7 @@ export class FinalizationHandler { /** * Drain a bounded number of durable finalized-SWM markers only while the * store scheduler reports no queued or in-flight work. The deterministic - * post-catch-up boundary may explicitly tolerate unrelated background work; + * post-catch-up boundary may explicitly queue cleanup behind active work; * periodic maintenance remains fully idle-only. * * The marker survives restart, while replacing the SWM head for a newer @@ -1621,19 +1621,19 @@ export class FinalizationHandler { contextGraphId: string; swmMetaGraph: string; maxCandidates?: number; - allowDuringBackgroundPressure?: boolean; + queueBehindActiveWork?: boolean; }): Promise { if (!this.writeLocks) return 0; const pressure = this.store.getPressureSnapshot?.(); - if (pressure && ( + if (!input.queueBehindActiveWork && pressure && ( pressure.ackInflight > 0 || (pressure.healthInflight ?? 0) > 0 || pressure.normalInflight > 0 - || (!input.allowDuringBackgroundPressure && pressure.backgroundInflight > 0) + || pressure.backgroundInflight > 0 || pressure.ackQueued > 0 || (pressure.healthQueued ?? 0) > 0 || pressure.normalQueued > 0 - || (!input.allowDuringBackgroundPressure && pressure.backgroundQueued > 0) + || pressure.backgroundQueued > 0 )) { return 0; } @@ -1663,15 +1663,15 @@ export class FinalizationHandler { let cleared = 0; for (const row of result.bindings) { const currentPressure = this.store.getPressureSnapshot?.(); - if (currentPressure && ( + if (!input.queueBehindActiveWork && currentPressure && ( currentPressure.ackInflight > 0 || (currentPressure.healthInflight ?? 0) > 0 || currentPressure.normalInflight > 0 - || (!input.allowDuringBackgroundPressure && currentPressure.backgroundInflight > 0) + || currentPressure.backgroundInflight > 0 || currentPressure.ackQueued > 0 || (currentPressure.healthQueued ?? 0) > 0 || currentPressure.normalQueued > 0 - || (!input.allowDuringBackgroundPressure && currentPressure.backgroundQueued > 0) + || currentPressure.backgroundQueued > 0 )) { break; } diff --git a/packages/agent/test/agent.part-16.test.ts b/packages/agent/test/agent.part-16.test.ts index 7fcc80d54c..0bc2b1e26f 100644 --- a/packages/agent/test/agent.part-16.test.ts +++ b/packages/agent/test/agent.part-16.test.ts @@ -203,7 +203,7 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => finalizedOnly: true, contextGraphIds: ['runtime-contextGraph'], finalizedCleanupBudget: 64, - allowDuringBackgroundPressure: true, + queueBehindActiveWork: true, }, ]]); expect(lifecycleOrder).toEqual(['durable', 'shared-memory', 'cleanup']); diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 5f393a1db2..1e5f40a160 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2220,20 +2220,20 @@ describe('graph-scoped finalization handler', () => { }); }); - it('does not starve finalized cleanup behind unrelated background sync pressure', async () => { + it('queues explicit post-catchup cleanup behind active store work', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); Object.defineProperty(store, 'getPressureSnapshot', { configurable: true, value: () => ({ - ackInflight: 0, - healthInflight: 0, - normalInflight: 0, + ackInflight: 1, + healthInflight: 1, + normalInflight: 2, backgroundInflight: 2, - ackQueued: 0, - healthQueued: 0, - normalQueued: 0, + ackQueued: 3, + healthQueued: 2, + normalQueued: 4, backgroundQueued: 4, maxConcurrent: 4, ackReservedSlots: 1, @@ -2245,7 +2245,7 @@ describe('graph-scoped finalization handler', () => { contextGraphId: CG, swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), maxCandidates: 16, - allowDuringBackgroundPressure: true, + queueBehindActiveWork: true, })).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); }); From f4ddb0138a377a5056358c8fed2f78c14699843c Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 17:48:20 +0200 Subject: [PATCH 06/48] fix(agent): retain explicit public CG cleanup scope --- packages/agent/src/dkg-agent-lifecycle.ts | 12 ++++---- .../agent/test/swm-ttl-v2-cleanup.test.ts | 28 ++++++++++++++++++- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 869499fb89..c75af66083 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -7609,11 +7609,13 @@ export class LifecycleSyncMethods extends DKGAgentBase { try { const graphManager = new GraphManager(this.store); - const requestedContextGraphs = options?.contextGraphIds - ? new Set(options.contextGraphIds) - : undefined; - const contextGraphs = (await graphManager.listContextGraphs()) - .filter((contextGraphId) => !requestedContextGraphs || requestedContextGraphs.has(contextGraphId)); + // A deterministic caller already knows the exact CG IDs. Do not route + // those IDs through GraphManager.listContextGraphs(): that storage-level + // helper intentionally omits owner/name public IDs because it only + // recognizes legacy flat graph IDs. + const contextGraphs = options?.contextGraphIds + ? [...new Set(options.contextGraphIds)] + : await graphManager.listContextGraphs(); for (const pid of contextGraphs) { let graphDeleted = 0; diff --git a/packages/agent/test/swm-ttl-v2-cleanup.test.ts b/packages/agent/test/swm-ttl-v2-cleanup.test.ts index b9caa1bb42..79dccdaf7d 100644 --- a/packages/agent/test/swm-ttl-v2-cleanup.test.ts +++ b/packages/agent/test/swm-ttl-v2-cleanup.test.ts @@ -10,7 +10,7 @@ * resolveKnowledgeAssetWorkspaceHead), * 3. the operation's public snapshot graph. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { makeTestKaNumberAllocator } from './_helpers/ka-allocator.js'; import { DKGAgent } from '../src/index.js'; import { createEVMAdapter, getSharedContext, createProvider, takeSnapshot, revertSnapshot, HARDHAT_KEYS } from '../../chain/test/evm-test-context.js'; @@ -236,4 +236,30 @@ describe('SWM TTL cleanup of graph-scoped V2 operations', () => { expect(await graphTripleCount(store, assertionGraph)).toBe(0); expect(await graphTripleCount(store, seeded.snapshotGraph)).toBe(0); }, 60_000); + + it('honors an explicit public owner/name context graph during finalized cleanup', async () => { + const cg = '0x1111111111111111111111111111111111111111/public-finalized-cleanup'; + const cleanupFinalizedGraphScopedSwmWhenIdle = vi.fn().mockResolvedValue(0); + const handlerSpy = vi + .spyOn(node as unknown as { getOrCreateFinalizationHandler: () => unknown }, 'getOrCreateFinalizationHandler') + .mockReturnValue({ cleanupFinalizedGraphScopedSwmWhenIdle }); + + try { + await node.cleanupExpiredSharedMemory({ + finalizedOnly: true, + contextGraphIds: [cg], + finalizedCleanupBudget: 4, + queueBehindActiveWork: true, + }); + } finally { + handlerSpy.mockRestore(); + } + + expect(cleanupFinalizedGraphScopedSwmWhenIdle).toHaveBeenCalledWith({ + contextGraphId: cg, + swmMetaGraph: contextGraphSharedMemoryMetaUri(cg), + maxCandidates: 4, + queueBehindActiveWork: true, + }); + }); }); From ba051602ab7b5adaf3d79297d89ce5d4d73d297a Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 18:24:10 +0200 Subject: [PATCH 07/48] fix(agent): retry finalized SWM drain under load --- packages/agent/src/finalization-handler.ts | 43 ++++++++++++++++--- .../ka-graph-finalization-handler.test.ts | 35 ++++++++++++++- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index ef7fe62522..ec3504514c 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -62,6 +62,8 @@ import { } from '@origintrail-official/dkg-publisher'; const DKG_NS = 'http://dkg.io/ontology/'; const PROV_NS = 'http://www.w3.org/ns/prov#'; +const FINALIZED_SWM_CLEANUP_BUSY_RETRY_BUDGET_MS = 120_000; +const FINALIZED_SWM_CLEANUP_BUSY_RETRY_DELAY_MS = 250; // Slow-query / canary tags for the finalization SWM slice (#1549). A healthy fleet // sees `.fallbackUnbounded` at ~0 relative to `.bounded`; a spike means the bound is @@ -1642,7 +1644,31 @@ export class FinalizationHandler { priority: 'background', source: 'agent.finalization.graphScopedSwmCleanup.discover', }; - const result = await this.store.query( + const busyRetryDeadline = input.queueBehindActiveWork + ? Date.now() + FINALIZED_SWM_CLEANUP_BUSY_RETRY_BUDGET_MS + : 0; + const runStoreOperation = async (operation: () => Promise): Promise => { + for (;;) { + try { + return await operation(); + } catch (error) { + if ( + !(error instanceof StoreSchedulerBusyError) + || !input.queueBehindActiveWork + || Date.now() >= busyRetryDeadline + ) { + throw error; + } + await new Promise((resolve) => { + setTimeout(resolve, Math.min( + FINALIZED_SWM_CLEANUP_BUSY_RETRY_DELAY_MS, + Math.max(1, busyRetryDeadline - Date.now()), + )); + }); + } + } + }; + const result = await runStoreOperation(() => this.store.query( `SELECT DISTINCT ?head ?ual ?version ?root ?shareId ?subGraphName WHERE { GRAPH <${assertSafeIri(input.swmMetaGraph)}> { ?head <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; @@ -1657,7 +1683,7 @@ export class FinalizationHandler { } } ORDER BY ?head LIMIT ${limit}`, background, - ); + )); if (result.type !== 'bindings') return 0; let cleared = 0; @@ -1698,16 +1724,19 @@ export class FinalizationHandler { const graphManager = new GraphManager(this.store); let expectedHead: KnowledgeAssetWorkspaceHead | undefined; try { - expectedHead = await resolveKnowledgeAssetWorkspaceHead({ + expectedHead = await runStoreOperation(() => resolveKnowledgeAssetWorkspaceHead({ store: this.store, graphManager, contextGraphId: input.contextGraphId, kaUal: scope.ual, subGraphName, queryOptions: background, - }); + })); } catch (error) { - if (error instanceof StoreSchedulerBusyError) break; + if (error instanceof StoreSchedulerBusyError) { + if (input.queueBehindActiveWork) throw error; + break; + } continue; } if ( @@ -1725,7 +1754,7 @@ export class FinalizationHandler { } catch { continue; } - const outcome = await this.clearMarkedFinalizedGraphScopedSwm({ + const outcome = await runStoreOperation(() => this.clearMarkedFinalizedGraphScopedSwm({ contextGraphId: input.contextGraphId, scope, expectedHead, @@ -1733,7 +1762,7 @@ export class FinalizationHandler { privateMerkleRoot, subGraphName, ctx: createOperationContext('system'), - }); + })); if (outcome === 'cleared' || outcome === 'absent') cleared += 1; } return cleared; diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 1e5f40a160..f329c64612 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -2250,6 +2250,39 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(0); }); + it('retries an explicit post-catchup cleanup after a transient scheduler timeout', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + const originalQuery = store.query.bind(store); + let injectedBusyTimeout = false; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if ( + !injectedBusyTimeout + && options?.source === 'agent.finalization.graphScopedSwmCleanup.discover' + && query.includes('SELECT ?scopeVersion ?kaUal ?assertionVersion') + ) { + injectedBusyTimeout = true; + throw new StoreSchedulerBusyError( + 'queue_wait_timeout', + 'background', + options.source, + ); + } + return originalQuery(query, options); + }); + + await expect(handler.cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + queueBehindActiveWork: true, + })).resolves.toBe(1); + expect(injectedBusyTimeout).toBe(true); + expect(await store.countQuads(swmGraph)).toBe(0); + querySpy.mockRestore(); + }); + it('repairs VM from the immutable operation snapshot after deferred SWM cleanup', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); From 1df66a3b9e7a22e72ad2aea60bb9aa11ffad78df Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 18:50:25 +0200 Subject: [PATCH 08/48] fix(agent): reserve lifecycle lane for SWM drain --- packages/agent/src/dkg-agent-lifecycle.ts | 12 ++++---- packages/agent/src/finalization-handler.ts | 30 ++++++++++++------- .../ka-graph-finalization-handler.test.ts | 8 ++++- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index c75af66083..67ca3a726d 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -6042,12 +6042,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (includeSharedMemory && typeof this.cleanupExpiredSharedMemory === 'function') { // Every selected peer has settled, so this is the first single, // deterministic cleanup boundary for the whole catch-up job. The - // cleanup's store operations remain background-priority and its - // per-KA lock/exact VM+SWM+marker re-checks preserve a concurrent newer - // SWM lifecycle. Queue behind active store work instead of sampling - // pressure and abandoning the drain: the store scheduler keeps - // foreground/ACK priority and guarantees background progress. The - // periodic timer remains the restart/failure backstop. + // cleanup's per-KA lock/exact VM+SWM+marker re-checks preserve a + // concurrent newer SWM lifecycle. Queue this bounded deterministic drain + // in the normal maintenance lane: ACK/health reservations remain + // protected, but a continuously deep background sync queue cannot make + // every cleanup attempt expire before admission. The periodic timer + // remains idle-gated/background and is the restart/failure backstop. await this.cleanupExpiredSharedMemory({ finalizedOnly: true, contextGraphIds: [contextGraphId], diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index ec3504514c..c81a0bb696 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -1640,8 +1640,13 @@ export class FinalizationHandler { return 0; } const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); - const background: QueryOptions = { - priority: 'background', + // A deterministic post-catch-up drain is part of completing the lifecycle + // job, not opportunistic maintenance. Give it the normal lane so a + // continuously deep background sync queue cannot make every retry expire + // before admission. ACK and health work retain their reserved slots, while + // periodic cleanup stays background-only and idle-gated. + const cleanupQueryOptions: QueryOptions = { + priority: input.queueBehindActiveWork ? 'normal' : 'background', source: 'agent.finalization.graphScopedSwmCleanup.discover', }; const busyRetryDeadline = input.queueBehindActiveWork @@ -1682,7 +1687,7 @@ export class FinalizationHandler { OPTIONAL { ?operation <${DKG_NS}subGraphName> ?subGraphName } } } ORDER BY ?head LIMIT ${limit}`, - background, + cleanupQueryOptions, )); if (result.type !== 'bindings') return 0; @@ -1730,7 +1735,7 @@ export class FinalizationHandler { contextGraphId: input.contextGraphId, kaUal: scope.ual, subGraphName, - queryOptions: background, + queryOptions: cleanupQueryOptions, })); } catch (error) { if (error instanceof StoreSchedulerBusyError) { @@ -1761,6 +1766,7 @@ export class FinalizationHandler { expectedMerkleRoot, privateMerkleRoot, subGraphName, + queryPriority: cleanupQueryOptions.priority, ctx: createOperationContext('system'), })); if (outcome === 'cleared' || outcome === 'absent') cleared += 1; @@ -1776,6 +1782,7 @@ export class FinalizationHandler { expectedMerkleRoot: Uint8Array; privateMerkleRoot?: Uint8Array; subGraphName?: string; + queryPriority?: QueryOptions['priority']; ctx: OperationContext; }): Promise<'cleared' | 'absent' | 'preserved'> { if (!this.writeLocks) return 'preserved'; @@ -1786,13 +1793,14 @@ export class FinalizationHandler { expectedMerkleRoot, privateMerkleRoot, subGraphName, + queryPriority, ctx, } = input; const lockKey = swmKaWriteLockKey(contextGraphId, subGraphName, scope.ual); const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { const graphManager = new GraphManager(this.store); - const background: QueryOptions = { - priority: 'background', + const cleanupQueryOptions: QueryOptions = { + priority: queryPriority ?? 'background', source: 'agent.finalization.graphScopedSwmCleanup', }; let currentHead: KnowledgeAssetWorkspaceHead | undefined; @@ -1803,7 +1811,7 @@ export class FinalizationHandler { contextGraphId, kaUal: scope.ual, subGraphName, - queryOptions: background, + queryOptions: cleanupQueryOptions, }); } catch (error) { if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; @@ -1832,7 +1840,7 @@ export class FinalizationHandler { `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` + `<${assertSafeIri(headSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` + `${cleanupRootObject} } }`, - background, + cleanupQueryOptions, ); if (marker.type !== 'boolean' || !marker.value) return 'preserved' as const; @@ -1845,7 +1853,7 @@ export class FinalizationHandler { expectedMerkleRoot, expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, subGraphName, - queryOptions: background, + queryOptions: cleanupQueryOptions, }); if (vmVerification.status !== 'verified') { this.log.warn( @@ -1865,7 +1873,7 @@ export class FinalizationHandler { expectedMerkleRoot, expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, subGraphName, - queryOptions: background, + queryOptions: cleanupQueryOptions, }); if ( swmVerification.status !== 'verified' @@ -1886,7 +1894,7 @@ export class FinalizationHandler { metaGraph, headSubject, [], - background, + cleanupQueryOptions, ); if (!replaced) { throw Object.assign( diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index f329c64612..bcf9108211 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2256,7 +2256,11 @@ describe('graph-scoped finalization handler', () => { const originalQuery = store.query.bind(store); let injectedBusyTimeout = false; + const cleanupPriorities: Array = []; const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source?.startsWith('agent.finalization.graphScopedSwmCleanup')) { + cleanupPriorities.push(options.priority); + } if ( !injectedBusyTimeout && options?.source === 'agent.finalization.graphScopedSwmCleanup.discover' @@ -2265,7 +2269,7 @@ describe('graph-scoped finalization handler', () => { injectedBusyTimeout = true; throw new StoreSchedulerBusyError( 'queue_wait_timeout', - 'background', + 'normal', options.source, ); } @@ -2279,6 +2283,8 @@ describe('graph-scoped finalization handler', () => { queueBehindActiveWork: true, })).resolves.toBe(1); expect(injectedBusyTimeout).toBe(true); + expect(cleanupPriorities.length).toBeGreaterThan(0); + expect(new Set(cleanupPriorities)).toEqual(new Set(['normal'])); expect(await store.countQuads(swmGraph)).toBe(0); querySpy.mockRestore(); }); From 84353e2b4ad0759307415d5e8027d291e9d85da3 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 30 Jul 2026 19:20:42 +0200 Subject: [PATCH 09/48] fix(agent): prevent finalized SWM resurrection --- .../src/sync/requester/shared-memory-sync.ts | 17 ++ .../requester/swm-snapshot-materializer.ts | 194 ++++++++++++++++-- ...wm-public-snapshot-materialization.test.ts | 25 +++ .../test/swm-snapshot-materializer.test.ts | 128 ++++++++++++ 4 files changed, 350 insertions(+), 14 deletions(-) diff --git a/packages/agent/src/sync/requester/shared-memory-sync.ts b/packages/agent/src/sync/requester/shared-memory-sync.ts index 191161845e..ee5717a53b 100644 --- a/packages/agent/src/sync/requester/shared-memory-sync.ts +++ b/packages/agent/src/sync/requester/shared-memory-sync.ts @@ -317,6 +317,23 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // re-checks below stop the other failure the lock alone cannot: // replacing newer content with an older verified snapshot. // + // (0) Finalized anti-resurrection. The deferred cleanup may + // have drained this exact assertion while another peer's SWM + // sync was still in flight. VM is the authoritative proof: + // once this descriptor is exactly confirmed there, never + // restore it to SWM. The materializer also removes an exact + // active duplicate atomically, while preserving a newer or + // otherwise different SWM lifecycle. + if (await snapshotMaterializer.discardFinalizedGraphAsset(pid, descriptor)) { + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); + } + materializedKeys.add(graphKey); + logDebug(ctx, `SWM sync for "${pid}": snapshot ${snapshotRef} is already ` + + 'exactly confirmed in VM; refusing SWM restoration'); + return; + } + // // (a) Version ordering. A stored head newer than the descriptor // means gossip advanced this KA past our snapshot; replacing // would be overwrite-with-older, byte-for-byte the regression diff --git a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts index 7277fb8f50..6701bdd097 100644 --- a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts +++ b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts @@ -15,13 +15,20 @@ */ import { assertSafeIri } from '@origintrail-official/dkg-core'; import { + computeFlatKCRootV10, + readConfirmedGraphKnowledgeAssetMetadataEnvelope, resolveKnowledgeAssetWorkspaceHead, sameKnowledgeAssetWorkspaceHead, swmKaWriteLockKey, withKeyedLocks, workspacePublicQuadsDigest, } from '@origintrail-official/dkg-publisher'; -import { GraphManager, type Quad, type TripleStore } from '@origintrail-official/dkg-storage'; +import { + GraphManager, + tryReplaceGraphAndSubjectAtomically, + type Quad, + type TripleStore, +} from '@origintrail-official/dkg-storage'; import type { GraphScopedSwmRecoveryDescriptor } from '../graph-scoped-swm-recovery.js'; import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from '../../dkg-agent-constants.js'; @@ -87,6 +94,20 @@ export interface SharedMemorySnapshotMaterializer { * of equal size apart and would skip a verified newer snapshot. */ isGraphAssetMaterialized(descriptor: GraphScopedSwmRecoveryDescriptor): Promise; + /** + * Refuse to resurrect an assertion that is already exactly confirmed in VM. + * When the active SWM head still names this descriptor, remove its exact + * duplicate graph and head atomically. A newer/different SWM head is + * preserved, but the stale finalized descriptor is still rejected. + * + * Returns true when the incoming descriptor is already exactly finalized, + * whether the local duplicate was removed, already absent, or preserved + * because the active SWM lifecycle differs. + */ + discardFinalizedGraphAsset( + contextGraphId: string, + descriptor: GraphScopedSwmRecoveryDescriptor, + ): Promise; /** * Atomic whole-graph replace. Replace, not insert: a KA graph is * all-or-nothing and digest-verified; union-insert risks partial or @@ -205,19 +226,10 @@ async function readExactFinalizedCleanupMarkers(params: { source: `${sourcePrefix}.readFinalizedCleanupHead`, }, }); - if (!currentHead || !sameKnowledgeAssetWorkspaceHead(currentHead, { - kaUal: descriptor.kaUal, - assertionVersion: descriptor.assertionVersion, - assertionGraph: descriptor.assertionGraph, - publicQuadsDigest: descriptor.publicQuadsDigest, - publicTripleCount: descriptor.publicQuadsCount, - privateMerkleRoot: descriptor.privateMerkleRoot, - privateTripleCount: descriptor.privateTripleCount, - shareOperationId: descriptor.shareOperationId, - publisherPeerId: descriptor.publisherPeerId, - accessPolicy: descriptor.accessPolicy, - allowedPeers: [...descriptor.allowedPeers], - })) { + if (!currentHead || !sameKnowledgeAssetWorkspaceHead( + currentHead, + workspaceHeadFromDescriptor(descriptor), + )) { return []; } } catch { @@ -254,6 +266,85 @@ async function readExactFinalizedCleanupMarkers(params: { return [headMarkers[0]!, operationMarkers[0]!]; } +function workspaceHeadFromDescriptor( + descriptor: GraphScopedSwmRecoveryDescriptor, +) { + return { + kaUal: descriptor.kaUal, + assertionVersion: descriptor.assertionVersion, + assertionGraph: descriptor.assertionGraph, + publicQuadsDigest: descriptor.publicQuadsDigest, + publicTripleCount: descriptor.publicQuadsCount, + privateMerkleRoot: descriptor.privateMerkleRoot, + privateTripleCount: descriptor.privateTripleCount, + shareOperationId: descriptor.shareOperationId, + publisherPeerId: descriptor.publisherPeerId, + accessPolicy: descriptor.accessPolicy, + allowedPeers: [...descriptor.allowedPeers], + }; +} + +function sameBytes(left: Uint8Array | undefined, right: Uint8Array | undefined): boolean { + if (left === undefined || right === undefined) return left === right; + return left.length === right.length && left.every((byte, index) => byte === right[index]); +} + +function descriptorPrivateRoot( + descriptor: GraphScopedSwmRecoveryDescriptor, +): Uint8Array | undefined { + const value = descriptor.privateMerkleRoot?.replace(/^0x/i, ''); + if (value === undefined) return undefined; + if (!/^[0-9a-fA-F]{64}$/.test(value)) return new Uint8Array(0); + return Uint8Array.from(value.match(/.{2}/g)!.map((pair) => Number.parseInt(pair, 16))); +} + +async function isExactConfirmedVmAsset(params: { + store: TripleStore; + contextGraphId: string; + descriptor: GraphScopedSwmRecoveryDescriptor; +}): Promise { + const { store, contextGraphId, descriptor } = params; + const confirmed = await readConfirmedGraphKnowledgeAssetMetadataEnvelope(store, { + contextGraphId, + ual: descriptor.kaUal, + }); + if (confirmed.state !== 'confirmed') return false; + const { envelope } = confirmed; + const expectedPrivateRoot = descriptorPrivateRoot(descriptor); + if ( + envelope.assertionVersion !== descriptor.assertionVersion + || envelope.publicTripleCount !== descriptor.publicQuadsCount + || envelope.privateTripleCount !== descriptor.privateTripleCount + || envelope.subGraphName !== descriptor.subGraphName + || !sameBytes(envelope.privateMerkleRoot, expectedPrivateRoot) + ) { + return false; + } + + const vmResult = await store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(envelope.assertionGraph)}> { ?s ?p ?o } }`, + { + priority: 'background', + source: 'agent.sharedMemorySync.snapshotMaterializer.readConfirmedVmGraph', + }, + ); + if (vmResult.type !== 'quads') return false; + const vmQuads = vmResult.quads.map((quad) => ({ ...quad, graph: '' })); + if ( + vmQuads.length !== descriptor.publicQuadsCount + || workspacePublicQuadsDigest(vmQuads) !== descriptor.publicQuadsDigest + ) { + return false; + } + return sameBytes( + computeFlatKCRootV10( + vmQuads, + envelope.privateMerkleRoot ? [envelope.privateMerkleRoot] : [], + ), + envelope.merkleRoot, + ); +} + /** * Build the production materializer over the agent's own store, lock map and * list-cache invalidation hook. @@ -327,6 +418,81 @@ export function createSharedMemorySnapshotMaterializer(deps: { return workspacePublicQuadsDigest(stored) === descriptor.publicQuadsDigest; }, + discardFinalizedGraphAsset: async (contextGraphId, descriptor) => { + if (!await isExactConfirmedVmAsset({ + store: deps.store, + contextGraphId, + descriptor, + })) { + return false; + } + + // The caller owns the canonical per-KA writer lock. Re-read the active + // head inside it so a newer SWM assertion can never be removed by a + // delayed finalized snapshot. + let currentHead; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: deps.store, + graphManager: new GraphManager(deps.store), + contextGraphId, + kaUal: descriptor.kaUal, + subGraphName: descriptor.subGraphName, + queryOptions: { + priority: 'background', + source: 'agent.sharedMemorySync.snapshotMaterializer.readFinalizedSwmHead', + }, + }); + } catch { + // Exact VM still makes the incoming descriptor stale. Preserve corrupt + // local SWM state for explicit recovery, but never import another copy. + return true; + } + if ( + !currentHead + || !sameKnowledgeAssetWorkspaceHead( + currentHead, + workspaceHeadFromDescriptor(descriptor), + ) + ) { + return true; + } + + const swmResult = await deps.store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(descriptor.assertionGraph)}> { ?s ?p ?o } }`, + { + priority: 'background', + source: 'agent.sharedMemorySync.snapshotMaterializer.readFinalizedSwmGraph', + }, + ); + if (swmResult.type !== 'quads') return true; + const swmQuads = swmResult.quads.map((quad) => ({ ...quad, graph: '' })); + const swmIsAbsent = swmQuads.length === 0; + const swmIsExact = swmQuads.length === descriptor.publicQuadsCount + && workspacePublicQuadsDigest(swmQuads) === descriptor.publicQuadsDigest; + if (!swmIsAbsent && !swmIsExact) return true; + + const replaced = await tryReplaceGraphAndSubjectAtomically( + deps.store, + descriptor.assertionGraph, + [], + descriptor.metaGraph, + descriptor.headSubject, + [], + { + priority: 'background', + source: 'agent.sharedMemorySync.snapshotMaterializer.discardFinalizedSwm', + }, + ); + if (!replaced) { + throw new Error( + 'finalized SWM anti-resurrection requires atomic graph-and-head replacement support', + ); + } + deps.invalidateListContextGraphsCache(); + return true; + }, + replaceGraph: async (graphUri, quads) => { // Deliberately NOT routed through the sync lane's guarded union insert: // a KA graph is all-or-nothing and digest-verified, so it must land via diff --git a/packages/agent/test/swm-public-snapshot-materialization.test.ts b/packages/agent/test/swm-public-snapshot-materialization.test.ts index d5f2dcf13e..d9db357d15 100644 --- a/packages/agent/test/swm-public-snapshot-materialization.test.ts +++ b/packages/agent/test/swm-public-snapshot-materialization.test.ts @@ -117,6 +117,7 @@ function fixture(subGraphName?: string) { interface HarnessOverrides { storedHead?: () => StoredWorkspaceHeadState; contentPresent?: () => boolean; + finalized?: () => boolean; replaceImpl?: (graphUri: string, quads: Quad[]) => Promise; onLockRequested?: () => void; lockMap?: Map>; @@ -188,6 +189,10 @@ function harness(overrides: HarnessOverrides = {}) { events.push('content-checked'); return overrides.contentPresent?.() ?? false; }, + discardFinalizedGraphAsset: async () => { + events.push('finalized-checked'); + return overrides.finalized?.() ?? false; + }, readStoredHead: async () => { events.push('version-read'); return overrides.storedHead?.() ?? { version: null, needsRepair: false }; @@ -308,6 +313,26 @@ describe('public SWM snapshot materialization', () => { expect(summary.failedPhases).toBe(0); }); + it('does not resurrect a snapshot that is already exactly finalized in VM', async () => { + // This is the late-sync race from the live blackbox run: cleanup drains + // SWM, then an already in-flight peer snapshot arrives. Exact VM proof + // makes that descriptor terminal, so neither data nor head metadata may + // be restored. + const h = harness({ + finalized: () => true, + storedHead: () => ({ version: '1', needsRepair: false }), + contentPresent: () => false, + }); + const summary = await h.run(); + expect(h.events).toContain('finalized-checked'); + expect(h.events).not.toContain('version-read'); + expect(h.events).not.toContain('content-checked'); + expect(h.events).not.toContain('replaced'); + expect(h.events).not.toContain('head-swapped'); + expect(h.inserted.every((batch) => batch.every((q) => q.graph !== WS_META))).toBe(true); + expect(summary.failedPhases).toBe(0); + }); + it('collapses union-insert residue on the skip path when the head needs repair', async () => { // Content already matches the descriptor, but the head subject carries // several version/operation rows (e.g. a prior round failed between the diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index 393d90a5fc..cbf7489730 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -31,6 +31,8 @@ import { type OperationContext, } from '@origintrail-official/dkg-core'; import { + computeFlatKCRootV10, + generateGraphKnowledgeAssetMetadata, generateKnowledgeAssetShareMetadata, resolveKnowledgeAssetWorkspaceHead, workspacePublicQuadsDigest, @@ -112,6 +114,38 @@ function descriptorFor(fixture: typeof v1) { return descriptors[0]!; } +function confirmedVmFor(fixture: typeof v1) { + const scope = createGraphKnowledgeAssetScope(UAL, fixture.version); + const assertionGraph = knowledgeAssetLayerGraphUri(CG, MemoryLayer.VerifiableMemory, scope); + const merkleRoot = computeFlatKCRootV10(fixture.payload, []); + const agentAddress = BigInt('0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); + const batchId = (agentAddress << 96n) | 9n; + const meta = generateGraphKnowledgeAssetMetadata({ + contextGraphId: CG, + ual: UAL, + merkleRoot, + publisherPeerId: 'peer-source', + accessPolicy: 'public', + allowedPeers: [], + timestamp: new Date(0), + assertionVersion: fixture.version, + authorAddress: '0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + publicTripleCount: fixture.payload.length, + privateTripleCount: 0, + assertionGraph, + }, { + status: 'confirmed', + confirmation: { + kind: 'finalized-materialization', + provenance: { + batchId, + materializedVersion: { blockNumber: 123, txIndex: 0 }, + }, + }, + }); + return { assertionGraph, merkleRoot, meta }; +} + function materializerFor(store: TripleStore) { let invalidations = 0; const materializer = createSharedMemorySnapshotMaterializer({ @@ -327,6 +361,71 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', }); }); + it('atomically discards an exact SWM duplicate already confirmed in VM', async () => { + const store = new OxigraphStore(); + const vm = confirmedVmFor(v1); + await store.insert([ + ...v1.meta, + ...inGraph(v1.payload, v1.assertionGraph), + ...inGraph(v1.payload, vm.assertionGraph), + ...vm.meta, + ]); + const { materializer } = materializerFor(store); + const descriptor = descriptorFor(v1); + + const finalized = await materializer.withKaWriteLock(CG, undefined, UAL, () => + materializer.discardFinalizedGraphAsset(CG, descriptor)); + + expect(finalized).toBe(true); + expect(await materializer.isGraphAssetMaterialized(descriptor)).toBe(false); + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([]); + // Immutable operation metadata remains available for receipt/reorg + // recovery; only the active SWM graph and head are drained. + expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)) + .toEqual(['"op-v1"']); + }); + + it('rejects a finalized inbound descriptor without deleting a newer SWM lifecycle', async () => { + const store = new OxigraphStore(); + const vm = confirmedVmFor(v1); + await store.insert([ + ...v2.meta, + ...inGraph(v2.payload, v2.assertionGraph), + ...inGraph(v1.payload, vm.assertionGraph), + ...vm.meta, + ]); + const { materializer } = materializerFor(store); + + const finalized = await materializer.withKaWriteLock(CG, undefined, UAL, () => + materializer.discardFinalizedGraphAsset(CG, descriptorFor(v1))); + + expect(finalized).toBe(true); + expect(await materializer.isGraphAssetMaterialized(descriptorFor(v2))).toBe(true); + expect(await distinctObjects(store, WS_META, v2.headSubject, `${DKG}shareOperationId`)) + .toEqual(['"op-v2"']); + }); + + it('does not treat confirmed metadata as proof when the VM graph content differs', async () => { + const store = new OxigraphStore(); + const vm = confirmedVmFor(v1); + await store.insert([ + ...v1.meta, + ...inGraph(v1.payload, v1.assertionGraph), + ...inGraph(v2.payload, vm.assertionGraph), + ...vm.meta, + ]); + const { materializer } = materializerFor(store); + + const finalized = await materializer.withKaWriteLock(CG, undefined, UAL, () => + materializer.discardFinalizedGraphAsset(CG, descriptorFor(v1))); + + expect(finalized).toBe(false); + expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(true); + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual(['"op-v1"']); + }); + it('replaceGraph writes atomically and invalidates the list cache', async () => { const store = new OxigraphStore(); const { materializer, invalidations } = materializerFor(store); @@ -446,5 +545,34 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', expect(again.failedPhases).toBe(0); expect(h.replaceCalls()).toBe(1); }); + + it('does not restore an exact finalized assertion after cleanup or a late peer sync', async () => { + const store = new OxigraphStore(); + const vm = confirmedVmFor(v1); + await store.insert([ + ...v1.meta, + ...inGraph(v1.payload, v1.assertionGraph), + ...inGraph(v1.payload, vm.assertionGraph), + ...vm.meta, + ]); + const h = realHarness(store, v1); + + const first = await h.run(); + expect(first.failedPhases).toBe(0); + expect(h.replaceCalls()).toBe(0); + const { materializer } = materializerFor(store); + expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(false); + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([]); + + // A second identical peer snapshot is still refused: confirmed VM is a + // durable anti-resurrection proof, not a one-shot cleanup marker. + const again = await h.run(); + expect(again.failedPhases).toBe(0); + expect(h.replaceCalls()).toBe(0); + expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(false); + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([]); + }); }); }); From 79019ce992e98e9dbb4d9b3fa9b540498f49ac32 Mon Sep 17 00:00:00 2001 From: Bojan Date: Fri, 31 Jul 2026 09:46:18 +0200 Subject: [PATCH 10/48] fix(agent): address red Bug Bot findings --- packages/agent/src/dkg-agent-lifecycle.ts | 69 ++++++++++--------- packages/agent/src/finalization-handler.ts | 3 +- .../requester/swm-snapshot-materializer.ts | 21 ++++-- .../agent/src/sync/responder/graph-plan.ts | 48 ++++++++++++- .../ka-graph-finalization-handler.test.ts | 34 ++++++++- .../test/swm-snapshot-materializer.test.ts | 1 + packages/agent/test/swm-snapshot-sync.test.ts | 57 +++++++++++++++ .../sync-responder-swm-meta-ceiling.test.ts | 23 ++++++- .../test/sync-responder-swm-subgraphs.test.ts | 25 ++++++- 9 files changed, 238 insertions(+), 43 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 67ca3a726d..9dd855c134 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -5290,6 +5290,23 @@ export class LifecycleSyncMethods extends DKGAgentBase { return admission; }; + const insertSharedMemorySyncQuads = async (quads: readonly Quad[]) => { + // Oversize guard (OT-RFC-56): drop+tombstone protocol-violating + // literals BEFORE every peer-controlled SWM metadata insert, + // including graph-scoped head replacement, so the page cursor can + // advance instead of re-fetching the same poison row forever. + const inserted = await insertWithOversizeGuard( + (kept) => this.store.insert(kept, { + priority: 'background', + source: 'agent.sharedMemorySync.storeInsert', + }), + quads, + { recordDrops: (drops, seam) => this.oversizeTombstoneLog.record(drops, seam) }, + 'swm-sync', + ); + this.contextGraphMetaProjection.markDirtyFromQuads(inserted); + }; + const syncPublicContextGraph = (contextGraphId: string, remainingContextGraphs: number) => runSharedMemorySync({ ctx, remotePeerId, @@ -5331,22 +5348,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { store: this.store, writeLocks: this.writeLocks, invalidateListContextGraphsCache: () => this.invalidateListContextGraphsCache(), + insertReplacementMetadata: insertSharedMemorySyncQuads, }), - storeInsert: async (quads) => { - // Oversize guard (OT-RFC-56): drop+tombstone protocol-violating - // literals BEFORE insert so the SWM page cursor advances instead - // of the store throwing and the page re-fetching forever. - const inserted = await insertWithOversizeGuard( - (kept) => this.store.insert(kept, { - priority: 'background', - source: 'agent.sharedMemorySync.storeInsert', - }), - quads, - { recordDrops: (drops, seam) => this.oversizeTombstoneLog.record(drops, seam) }, - 'swm-sync', - ); - this.contextGraphMetaProjection.markDirtyFromQuads(inserted); - }, + storeInsert: insertSharedMemorySyncQuads, publicSnapshotStore: this.publicSnapshotStore, deleteCheckpoint: (key) => this.syncCheckpoints.delete(key), setCheckpoint: (key, offset) => this.syncCheckpoints.set(key, offset), @@ -7830,6 +7834,23 @@ async function runRecoverContextGraphSwmFromPeer( const admission = await getSharedMemorySubGraphAdmission( dependencies.store, contextGraphId, dependencies.listSubGraphs(contextGraphId), ); + const insertRecoveredSwmQuads = async (quads: readonly Quad[]) => { + // Oversize guard (OT-RFC-56) — recovered rows and graph-scoped + // replacement metadata are both peer-controlled data. + const inserted = await insertWithOversizeGuard( + (kept) => dependencies.store.insert(kept, { + priority: 'background', + source: 'agent.swmRecovery.insert', + }), + quads, + { recordDrops: (drops, seam) => dependencies.recordDrops(drops, seam) }, + 'swm-recovery', + ); + if (inserted.length > 0) { + dependencies.invalidateListContextGraphsCache(); + dependencies.markMetaProjectionDirty(inserted); + } + }; return recoverContextGraphSwm({ ctx, remotePeerId, @@ -7848,22 +7869,7 @@ async function runRecoverContextGraphSwmFromPeer( // dirty on insert (parity with runSharedMemorySync's // insertSyncedQuadsAndInvalidateListCache); deletes pass through to the store. store: { - insert: async (quads) => { - // Oversize guard (OT-RFC-56) — recovered rows are peer data too. - const inserted = await insertWithOversizeGuard( - (kept) => dependencies.store.insert(kept, { - priority: 'background', - source: 'agent.swmRecovery.insert', - }), - quads, - { recordDrops: (drops, seam) => dependencies.recordDrops(drops, seam) }, - 'swm-recovery', - ); - if (inserted.length > 0) { - dependencies.invalidateListContextGraphsCache(); - dependencies.markMetaProjectionDirty(inserted); - } - }, + insert: insertRecoveredSwmQuads, replaceGraph: async (graph, quads) => { const replaced = await tryReplaceGraphAtomically( dependencies.store, @@ -7957,6 +7963,7 @@ async function runRecoverContextGraphSwmFromPeer( contextGraphId, descriptor: asset, sourcePrefix: 'agent.swmRecovery.replaceMetaForGraphAssets', + insertReplacementMetadata: insertRecoveredSwmQuads, }), ); } diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index c81a0bb696..4c5783435f 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -1681,7 +1681,8 @@ export class FinalizationHandler { <${DKG_NS}assertionVersion> ?version ; <${DKG_NS}shareOperationId> ?shareId ; <${DKG_NS}assertionGraph> ?assertionGraph . - ?operation <${DKG_NS}shareOperationId> ?shareId ; + ?operation <${DKG_NS}WorkspaceOperation> ; + <${DKG_NS}shareOperationId> ?shareId ; <${DKG_NS}kaUal> ?ual ; <${DKG_NS}assertionVersion> ?version . OPTIONAL { ?operation <${DKG_NS}subGraphName> ?subGraphName } diff --git a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts index 6701bdd097..3abe67e164 100644 --- a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts +++ b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts @@ -150,8 +150,15 @@ export async function replaceGraphScopedSwmHeadMetadata(params: { contextGraphId: string; descriptor: GraphScopedSwmRecoveryDescriptor; sourcePrefix: string; + insertReplacementMetadata: (quads: readonly Quad[]) => Promise; }): Promise { - const { store, contextGraphId, descriptor, sourcePrefix } = params; + const { + store, + contextGraphId, + descriptor, + sourcePrefix, + insertReplacementMetadata, + } = params; const preservedMarkers = await readExactFinalizedCleanupMarkers({ store, contextGraphId, @@ -200,10 +207,7 @@ export async function replaceGraphScopedSwmHeadMetadata(params: { ...preservedMarkers, ]; if (replacementQuads.length > 0) { - await store.insert(replacementQuads, { - priority: 'background', - source: `${sourcePrefix}.insertReplacementMetadata`, - }); + await insertReplacementMetadata(replacementQuads); } } @@ -357,6 +361,12 @@ export function createSharedMemorySnapshotMaterializer(deps: { */ writeLocks: Map>; invalidateListContextGraphsCache: () => void; + /** + * The same filtered insert seam used by aggregate SWM pages. Replacement + * metadata is peer-controlled too, so bypassing this guard can re-arm the + * oversized-literal retry loop after the old head has been removed. + */ + insertReplacementMetadata: (quads: readonly Quad[]) => Promise; }): SharedMemorySnapshotMaterializer { return { withKaWriteLock: (contextGraphId, subGraphName, kaUal, fn) => @@ -513,6 +523,7 @@ export function createSharedMemorySnapshotMaterializer(deps: { contextGraphId, descriptor, sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + insertReplacementMetadata: deps.insertReplacementMetadata, }); deps.invalidateListContextGraphsCache(); }, diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 077e964cf4..8421b548ab 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -919,6 +919,14 @@ export async function readSwmDataPage(params: { }): Promise { const dataGraphs = swmGraphsForRegisteredSubGraphs(params.contextGraphId, params.registeredSubGraphNames, false); const graphSet = new Set(params.graphList); + const swmMetaGraphs = dataGraphs + .map((graph) => `${graph}_meta`) + .filter((graph) => graphSet.has(graph)); + const finalizedAssertionGraphs = readFinalizedSwmAssertionGraphs( + params.store, + swmMetaGraphs, + params.signal, + ); const candidateGraphsFor = (graph: string) => params.graphList .filter((candidate) => candidate === graph || isSharedMemoryBucketDescendantDataGraph(candidate, graph)) .sort(compareCodePoint); @@ -933,7 +941,10 @@ export async function readSwmDataPage(params: { : undefined; if (!params.cutoffIso) { - const candidateGraphs = dedupeStrings(dataGraphs.flatMap(candidateGraphsFor)).sort(compareCodePoint); + const blockedGraphs = await finalizedAssertionGraphs; + const candidateGraphs = dedupeStrings(dataGraphs.flatMap(candidateGraphsFor)) + .filter((graph) => !blockedGraphs.has(graph)) + .sort(compareCodePoint); return readPagedRowsAcrossGraphs( params.store, candidateGraphs, @@ -947,12 +958,13 @@ export async function readSwmDataPage(params: { } const loadStoreBoundedPage: StorePageLoader = async (offset, limit, signal) => { - const loadPlan = () => buildFreshSwmDataGraphPlan( + const loadPlan = async () => buildFreshSwmDataGraphPlan( params.store, dataGraphs, graphSet, candidateGraphsFor, params.cutoffIso!, + await finalizedAssertionGraphs, signal, ); const plan = params.freshGraphPlanMemo && params.rowListCacheKey @@ -982,6 +994,33 @@ export async function readSwmDataPage(params: { ); } +/** + * Read the assertion graphs whose active graph-scoped SWM lifecycle has been + * finalized and durably marked for deferred cleanup. The marker itself is + * filtered from the meta phase; this companion filter keeps the corresponding + * payload graph out of both the cutoff-less exact plan and TTL data plans. + */ +async function readFinalizedSwmAssertionGraphs( + store: TripleStore, + swmMetaGraphs: readonly string[], + signal?: AbortSignal, +): Promise> { + const values = graphValues(swmMetaGraphs); + if (!values) return new Set(); + // sparql-scan-allow: R2 -- ?metaGraph is bound by the finite admitted SWM meta graph family + const result = await store.query(` + SELECT DISTINCT ?assertionGraph WHERE { + VALUES ?metaGraph { ${values} } + GRAPH ?metaGraph { + ?marked <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; + <${DKG_ASSERTION_GRAPH}> ?assertionGraph . + } + } + `, syncResponderStoreOptions(signal, 'sync.responder.readFinalizedSwmAssertionGraphs')); + if (result.type !== 'bindings') return new Set(); + return new Set(result.bindings.map((row) => row['assertionGraph']).filter(Boolean)); +} + export async function readDurableMetaPage(params: { store: TripleStore; contextGraphId: string; @@ -3344,6 +3383,7 @@ async function buildFreshSwmDataGraphPlan( graphSet: ReadonlySet, candidateGraphsFor: (graph: string) => string[], cutoffIso: string, + finalizedAssertionGraphs: ReadonlySet, signal?: AbortSignal, ): Promise { const cutoffFilter = @@ -3358,7 +3398,9 @@ async function buildFreshSwmDataGraphPlan( } const uniqueCandidates = [...new Map( candidates.map((candidate) => [candidate.graph, candidate]), - ).values()].sort((a, b) => compareCodePoint(a.graph, b.graph)); + ).values()] + .filter((candidate) => !finalizedAssertionGraphs.has(candidate.graph)) + .sort((a, b) => compareCodePoint(a.graph, b.graph)); if (uniqueCandidates.length === 0) return { entries: [], totalRows: 0 }; const rootsByGraph = new Map>(); diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index bcf9108211..67ae297a6a 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2250,6 +2250,39 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(0); }); + it('discovers the WorkspaceOperation for bounded subgraph cleanup', async () => { + const subGraphName = 'named-cleanup'; + const { message, swmGraph } = await stageGraph(undefined, subGraphName); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + const discoverQueries: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalization.graphScopedSwmCleanup.discover') { + discoverQueries.push(query); + } + return originalQuery(query, options); + }); + + await expect(handler.cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG, subGraphName), + maxCandidates: 1, + })).resolves.toBe(1); + expect(discoverQueries.some((query) => query.includes( + ' ', + ))).toBe(true); + expect(await store.countQuads(swmGraph)).toBe(0); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + subGraphName, + })).resolves.toBeUndefined(); + querySpy.mockRestore(); + }); + it('retries an explicit post-catchup cleanup after a transient scheduler timeout', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2264,7 +2297,6 @@ describe('graph-scoped finalization handler', () => { if ( !injectedBusyTimeout && options?.source === 'agent.finalization.graphScopedSwmCleanup.discover' - && query.includes('SELECT ?scopeVersion ?kaUal ?assertionVersion') ) { injectedBusyTimeout = true; throw new StoreSchedulerBusyError( diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index cbf7489730..064fc3220e 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -152,6 +152,7 @@ function materializerFor(store: TripleStore) { store, writeLocks: new Map>(), invalidateListContextGraphsCache: () => { invalidations += 1; }, + insertReplacementMetadata: (quads) => store.insert([...quads]), }); return { materializer, invalidations: () => invalidations }; } diff --git a/packages/agent/test/swm-snapshot-sync.test.ts b/packages/agent/test/swm-snapshot-sync.test.ts index 306af03a8e..97feb827da 100644 --- a/packages/agent/test/swm-snapshot-sync.test.ts +++ b/packages/agent/test/swm-snapshot-sync.test.ts @@ -97,6 +97,63 @@ describe('SWM snapshot catch-up sync', () => { ); }); + it('drops oversized graph-scoped replacement metadata without retrying the sync forever', async () => { + const nodeADataDir = await tempDataDir(); + const nodeBDataDir = await tempDataDir(); + const nodeA = await createAgent(nodeADataDir, 'SnapshotSyncOversizeA'); + const nodeB = await createAgent(nodeBDataDir, 'SnapshotSyncOversizeB'); + const sourceSnapshots = new FileWorkspacePublicSnapshotStore(join(nodeADataDir, 'swm-public-snapshots')); + + const write = await nodeA.publisher.writeToWorkspace(CONTEXT_GRAPH, [ + { subject: ENTITY, predicate: 'http://schema.org/name', object: '"Oversize metadata guard"', graph: '' }, + ], { publisherPeerId: 'peer-a' }); + const metaGraph = contextGraphWorkspaceMetaGraphUri(CONTEXT_GRAPH); + const operationSubject = `urn:dkg:share:${CONTEXT_GRAPH}:${write.shareOperationId}`; + const publisherPeerPredicate = 'http://dkg.io/ontology/publisherPeerId'; + await nodeA.store.deleteByPattern({ + graph: metaGraph, + subject: operationSubject, + predicate: publisherPeerPredicate, + }); + const oversizedPeerId = 'x'.repeat(61_000); + await nodeA.store.insert([{ + graph: metaGraph, + subject: operationSubject, + predicate: publisherPeerPredicate, + object: `"${oversizedPeerId}"`, + }]); + + installSharedMemorySyncMock(nodeB, nodeA, sourceSnapshots); + const detailedSync = () => (nodeB as unknown as { + syncSharedMemoryFromPeerDetailed(peerId: string, contextGraphIds: string[]): Promise<{ + failedPeers: number; + failedPhases: number; + completedPhases: number; + }>; + }).syncSharedMemoryFromPeerDetailed(REMOTE_PEER, [CONTEXT_GRAPH]); + + await expect(detailedSync()).resolves.toMatchObject({ + failedPeers: 0, + failedPhases: 0, + }); + // A second complete round pins the no-loop property: the poisoned row is + // seen again, but the already materialized graph/head remains usable and + // the phase still completes instead of re-fetching forever. + await expect(detailedSync()).resolves.toMatchObject({ + failedPeers: 0, + failedPhases: 0, + }); + + await expect(nodeB.store.query( + `ASK { GRAPH <${metaGraph}> { ?head ` + + `"${write.shareOperationId}" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + await expect(nodeB.store.query( + `ASK { GRAPH <${metaGraph}> { <${operationSubject}> ` + + `<${publisherPeerPredicate}> "${oversizedPeerId}" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: false }); + }); + it('does not insert dangling publicSnapshotRef metadata when remote snapshots are unavailable', async () => { const nodeADataDir = await tempDataDir(); const nodeBDataDir = await tempDataDir(); diff --git a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts index 14d246bc0a..532f0d52e6 100644 --- a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts +++ b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts @@ -647,6 +647,22 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { for (const opId of ['x', 'y', 'z']) { await store.insert(workspaceOpQuads(cgId, opId, `urn:l:${opId}`, metaGraph, iso)); } + const markedUal = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/1'; + const unmarkedUal = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ac/1'; + const markedHead = `${markedUal}#dkg-swm-head`; + const markedOp = `urn:dkg:share:${cgId}:marked`; + const unmarkedHead = `${unmarkedUal}#dkg-swm-head`; + const unmarkedOp = `urn:dkg:share:${cgId}:unmarked`; + await store.insert([ + ...graphScopedHeadQuads(cgId, metaGraph, markedUal, 'marked', iso), + { + graph: metaGraph, + subject: markedHead, + predicate: `${DKG_NS}finalizedSwmCleanupRoot`, + object: `"0x${'ab'.repeat(32)}"`, + }, + ...graphScopedHeadQuads(cgId, metaGraph, unmarkedUal, 'unmarked', iso), + ]); let legacyPagedQueries = 0; const originalQuery = store.query.bind(store); @@ -679,7 +695,12 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { { contextGraphId: cgId, includeSharedMemory: true, phase: 'meta', limit: 4, syncSessionId: 'legacy' }, 4, ); - expect(lines.size).toBe(15); + const joined = [...lines].join('\n'); + expect(lines.size).toBe(26); + expect(joined).not.toContain(markedHead); + expect(joined).not.toContain(markedOp); + expect(joined).toContain(unmarkedHead); + expect(joined).toContain(unmarkedOp); expect(legacyPagedQueries).toBeGreaterThan(0); await store.close(); }); diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index 029747a129..32af15939a 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -418,6 +418,11 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { const opId = 'finalized-deferred-cleanup'; const op = `urn:dkg:share:${CG_ID}:${opId}`; const head = `${ual}#dkg-swm-head`; + const assertionGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ab/9`; + const markedEntity = 'urn:swm:finalized:must-not-sync'; + const unmarkedGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ac/10`; + const unmarkedRoot = 'urn:swm:unmarked:must-sync'; + const unmarkedOp = `urn:dkg:share:${CG_ID}:unmarked-data`; const tupleRows = (subject: string) => [ { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}contentScopeVersion`, object: '"2"^^' }, { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}kaUal`, object: ual }, @@ -429,8 +434,13 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, ...tupleRows(op), ...tupleRows(head), - { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}assertionGraph`, object: `${ROOT_SWM}/0x00000000000000000000000000000000000000ab/9` }, + { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: assertionGraph, subject: markedEntity, predicate: 'http://schema.org/name', object: '"finalized-copy"' }, + { graph: unmarkedGraph, subject: unmarkedRoot, predicate: 'http://schema.org/name', object: '"live-copy"' }, + { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, + { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: `${DKG_NS}rootEntity`, object: unmarkedRoot }, ]); const markedCap = captureHandler(); registerSyncHandler({ @@ -459,6 +469,19 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { expect(out).not.toContain(op); expect(out).not.toContain('finalizedSwmCleanupRoot'); + const dataOut = await markedCap.invoke({ + contextGraphId: CG_ID, + syncSessionId: `finalized-cleanup-data-${sharedMemoryTtlMs}`, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'data', + }); + expect(dataOut).not.toContain(markedEntity); + expect(dataOut).not.toContain('"finalized-copy"'); + expect(dataOut).toContain(unmarkedRoot); + expect(dataOut).toContain('"live-copy"'); + // After idle cleanup removes the active head, the immutable operation // keeps the marker for recovery but must remain outside sync. await markedStore.deleteByPattern({ From ab39b899ec61d7d1145221ecaa6a0ce2c05464cf Mon Sep 17 00:00:00 2001 From: Bojan Date: Fri, 31 Jul 2026 10:05:16 +0200 Subject: [PATCH 11/48] test(chain): allow parallel code-check ordering --- .../chain/test/strict-current-finalized-evm-rpc.unit.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/chain/test/strict-current-finalized-evm-rpc.unit.test.ts b/packages/chain/test/strict-current-finalized-evm-rpc.unit.test.ts index 808cfdb7e4..2c84182336 100644 --- a/packages/chain/test/strict-current-finalized-evm-rpc.unit.test.ts +++ b/packages/chain/test/strict-current-finalized-evm-rpc.unit.test.ts @@ -244,8 +244,9 @@ describe('RFC-64 strict current-finalized raw JSON-RPC transport', () => { 'eth_getCode', 'eth_getCode', ]); - expect(server.calls[2]!.params[0]).toBe(TO); - expect(server.calls[3]!.params[0]).toBe(OTHER_TO); + expect(server.calls.slice(2).map(({ params }) => params[0])).toEqual( + expect.arrayContaining([TO, OTHER_TO]), + ); }); it('rejects an oversized generic return only after a stable fallback sandwich', async () => { From 9958b67ff78b3ba2fd64e6293127aa8762deaad3 Mon Sep 17 00:00:00 2001 From: Bojan Date: Fri, 31 Jul 2026 11:18:24 +0200 Subject: [PATCH 12/48] test(agent): address finalized SWM review findings --- packages/agent/src/dkg-agent-lifecycle.ts | 28 ++++++++- .../agent/src/sync/responder/graph-plan.ts | 4 +- .../ka-graph-finalization-handler.test.ts | 46 +++++++++++++- .../test/sync-responder-swm-subgraphs.test.ts | 63 +++++++++++++++++++ packages/agent/test/workspace-ttl.test.ts | 34 +++++++++- packages/agent/vitest.unit.config.ts | 16 +++++ 6 files changed, 186 insertions(+), 5 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 9dd855c134..608ca0a5b0 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -98,7 +98,7 @@ import { pickNetworkTunables, withRetry, } from '@origintrail-official/dkg-core'; -import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type TripleStore, type TripleStoreConfig, type StorePressureSnapshot, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; import { readChangelogDeltaPage } from './sync/responder/graph-plan.js'; import { decodeChangelogRequest, encodeChangelogResponse } from './sync/changelog/wire.js'; import { runChangelogSync, planPageApply } from './sync/requester/changelog-sync.js'; @@ -373,6 +373,20 @@ type JoinApprovalRetryEntry = { nextAttemptAt: number; lastError: string; }; + +function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): boolean { + if (!snapshot) return false; + return [ + snapshot.ackInflight, + snapshot.healthInflight ?? 0, + snapshot.normalInflight, + snapshot.backgroundInflight, + snapshot.ackQueued, + snapshot.healthQueued ?? 0, + snapshot.normalQueued, + snapshot.backgroundQueued, + ].some((count) => count > 0); +} import { multiaddr } from '@multiformats/multiaddr'; import { buildCclPolicyQuads, buildPolicyApprovalQuads, buildPolicyRevocationQuads, hashCclPolicy, type CclPolicyRecord, type PolicyApprovalBinding } from './ccl-policy.js'; import { CclEvaluator, parseCclPolicy, validateCclPolicy, type CclEvaluationResult, type CclFactTuple } from './ccl-evaluator.js'; @@ -7602,6 +7616,18 @@ export class LifecycleSyncMethods extends DKGAgentBase { }): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; const ctx = createOperationContext('share'); + // TTL-disabled nodes still need bounded finalized-SWM maintenance, but an + // ordinary periodic tick must not start graph discovery while foreground + // store work is active. Explicit callers that name CGs or deliberately + // queue behind active work retain deterministic cleanup semantics. + if ( + ttl <= 0 + && !options?.contextGraphIds + && !options?.queueBehindActiveWork + && hasActiveStorePressure(this.store.getPressureSnapshot?.()) + ) { + return 0; + } const cutoff = !options?.finalizedOnly && ttl > 0 ? new Date(Date.now() - ttl).toISOString() : undefined; diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 8421b548ab..c47e59432a 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -2695,7 +2695,7 @@ async function readBoundedSwmMetaSnapshot( return filterSwmMetaSnapshotRows(rows, null); } -function filterSwmMetaSnapshotRows( +export function filterSwmMetaSnapshotRows( rows: readonly SyncRow[], cutoffIso: string | null, ): SyncRow[] { @@ -2709,6 +2709,7 @@ function filterSwmMetaSnapshotRows( (bySubject.get(subject) ?? []) .filter((row) => row.p === predicate) .map((row) => row.o); + const cutoffMs = cutoffIso == null ? Number.NaN : Date.parse(cutoffIso); const isFresh = (subject: string): boolean => objects(subject, DKG_PUBLISHED_AT) .some((value) => { const timestamp = Date.parse(stripLiteral(value)); @@ -2744,7 +2745,6 @@ function filterSwmMetaSnapshotRows( } const syncableRows = rows.filter((row) => !blockedSubjects.has(row.s)); if (cutoffIso == null) return syncableRows.sort(compareRows); - const cutoffMs = Date.parse(cutoffIso); if (!Number.isFinite(cutoffMs)) return []; const admitted = new Set(); diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 67ae297a6a..c999555c83 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2086,6 +2086,49 @@ describe('graph-scoped finalization handler', () => { expect(currentHead?.assertionVersion).toBe('2'); }); + it('preserves finalized SWM when the committed VM graph disappears before cleanup', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await store.countQuads(vmGraph)).toBe(2); + + // Simulate external VM loss only after finalization has persisted the + // cleanup marker. The drain must re-verify VM durability and fail closed. + await store.dropGraph(vmGraph); + + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); + }); + + it('preserves finalized SWM when its exact assertion changes before cleanup', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + // Keep the finalized rows but add one new row after the marker is written. + // Exact-digest re-verification must reject this three-row graph. + await store.insert([{ + subject: 'urn:asset:post-finalization-change', + predicate: 'urn:predicate:value', + object: '"newer"', + graph: swmGraph, + }]); + + expect(await store.countQuads(swmGraph)).toBe(3); + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(3); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); + }); + it('serializes finalized SWM cleanup with the shared per-KA writer lock', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); const writeLocks = new Map>(); @@ -2283,7 +2326,7 @@ describe('graph-scoped finalization handler', () => { querySpy.mockRestore(); }); - it('retries an explicit post-catchup cleanup after a transient scheduler timeout', async () => { + it('retries the actual post-catchup discover query after a transient scheduler timeout', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2297,6 +2340,7 @@ describe('graph-scoped finalization handler', () => { if ( !injectedBusyTimeout && options?.source === 'agent.finalization.graphScopedSwmCleanup.discover' + && query.includes('SELECT DISTINCT ?head ?ual ?version ?root ?shareId ?subGraphName') ) { injectedBusyTimeout = true; throw new StoreSchedulerBusyError( diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index 32af15939a..5c29ab6d77 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -1,6 +1,10 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { OxigraphStore } from '@origintrail-official/dkg-storage'; import { registerSyncHandler } from '../src/sync/responder/sync-handler.js'; +import { + filterSwmMetaSnapshotRows, + type SyncRow, +} from '../src/sync/responder/graph-plan.js'; import type { SyncRequestEnvelope } from '../src/sync/auth/request-build.js'; import type { OperationContext } from '@origintrail-official/dkg-core'; @@ -63,6 +67,65 @@ const REMOTE_PEER_ID = '12D3KooWSmU3owJvB9sFw8uApDgKrv2VBMecsGGvgAc4Gq6hB57M'; const noopLog = (_ctx: OperationContext, _msg: string) => {}; +describe('finalized SWM snapshot blocking', () => { + it('blocks a marked head and its operation sibling by their shared lifecycle tuple', () => { + const graph = 'did:dkg:context-graph:tuple-block/_shared_memory_meta'; + const ual = 'did:dkg:otp:20430/0x1111111111111111111111111111111111111111/7'; + const head = `${ual}#dkg-swm-head`; + const operation = 'urn:dkg:share:tuple-block:operation-1'; + const unrelated = 'urn:dkg:share:tuple-block:unrelated'; + const tupleRows = (subject: string): SyncRow[] => [ + { g: graph, s: subject, p: `${DKG_NS}contentScopeVersion`, o: '"2"^^' }, + { g: graph, s: subject, p: `${DKG_NS}kaUal`, o: ual }, + { g: graph, s: subject, p: `${DKG_NS}assertionVersion`, o: '"1"' }, + { g: graph, s: subject, p: `${DKG_NS}shareOperationId`, o: '"operation-1"' }, + ]; + const rows: SyncRow[] = [ + ...tupleRows(head), + { g: graph, s: head, p: `${DKG_NS}finalizedSwmCleanupRoot`, o: '"sha256:abc"' }, + ...tupleRows(operation), + { g: graph, s: unrelated, p: RDF_TYPE, o: `${DKG_NS}WorkspaceOperation` }, + ]; + + const filtered = filterSwmMetaSnapshotRows(rows, null); + + expect(filtered.some((row) => row.s === head)).toBe(false); + expect(filtered.some((row) => row.s === operation)).toBe(false); + expect(filtered).toEqual([{ + g: graph, + s: unrelated, + p: RDF_TYPE, + o: `${DKG_NS}WorkspaceOperation`, + }]); + }); + + it('applies a valid freshness cutoff without a declaration-order failure', () => { + const graph = 'did:dkg:context-graph:freshness/_shared_memory_meta'; + const subject = 'urn:dkg:share:freshness:operation-1'; + const rows: SyncRow[] = [{ + g: graph, + s: subject, + p: RDF_TYPE, + o: `${DKG_NS}WorkspaceOperation`, + }, { + g: graph, + s: subject, + p: `${DKG_NS}publishedAt`, + o: '"2026-07-31T09:00:00.000Z"^^', + }]; + + const filtered = filterSwmMetaSnapshotRows( + rows, + '2026-07-31T08:00:00.000Z', + ); + expect(filtered).toHaveLength(2); + expect(new Set(filtered.map((row) => row.p))).toEqual(new Set([ + RDF_TYPE, + `${DKG_NS}publishedAt`, + ])); + }); +}); + function captureHandler(): { register: (proto: string, h: (data: Uint8Array, peerId: string) => Promise) => void; invoke: (envelope: SyncRequestEnvelope) => Promise; diff --git a/packages/agent/test/workspace-ttl.test.ts b/packages/agent/test/workspace-ttl.test.ts index bef61bb4a6..5c6c7b526d 100644 --- a/packages/agent/test/workspace-ttl.test.ts +++ b/packages/agent/test/workspace-ttl.test.ts @@ -2,12 +2,13 @@ * Tests for workspace TTL / expiry: expired workspace operations are cleaned * up and not served to peers during sync. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { makeTestKaNumberAllocator } from "./_helpers/ka-allocator.js"; import { DKGAgent } from '../src/index.js'; import { createEVMAdapter, getSharedContext, createProvider, takeSnapshot, revertSnapshot, HARDHAT_KEYS } from '../../chain/test/evm-test-context.js'; import { mintTokens } from '../../chain/test/hardhat-harness.js'; import { ethers } from 'ethers'; +import type { TripleStore } from '@origintrail-official/dkg-storage'; let _fileSnapshot: string; beforeAll(async () => { @@ -112,6 +113,37 @@ describe('setSharedMemoryTtlMs maintenance timer lifecycle', () => { // workspace TTL expiry is disabled. expect((node as any).swmCleanupTimer).not.toBeNull(); + const store = (node as unknown as { store: TripleStore }).store; + if (!store.listGraphsByPrefix) throw new Error('test store must expose graph-prefix discovery'); + let busy = true; + const pressureSpy = vi.spyOn(store, 'getPressureSnapshot').mockImplementation(() => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + })); + const graphDiscoverySpy = vi.spyOn(store, 'listGraphsByPrefix'); + const discoveryCallsBeforeBusyTick = graphDiscoverySpy.mock.calls.length; + + // A TTL-disabled periodic-style call exits before graph discovery while + // foreground work is active. + expect(await node.cleanupExpiredSharedMemory()).toBe(0); + expect(graphDiscoverySpy).toHaveBeenCalledTimes(discoveryCallsBeforeBusyTick); + + // Once the store becomes idle, the same TTL-disabled maintenance path + // resumes graph discovery for deferred finalized-SWM cleanup. + busy = false; + await node.cleanupExpiredSharedMemory(); + expect(graphDiscoverySpy.mock.calls.length).toBeGreaterThan(discoveryCallsBeforeBusyTick); + pressureSpy.mockRestore(); + graphDiscoverySpy.mockRestore(); + // Enable TTL at runtime node.setSharedMemoryTtlMs(60_000); expect((node as any).swmCleanupTimer).not.toBeNull(); diff --git a/packages/agent/vitest.unit.config.ts b/packages/agent/vitest.unit.config.ts index f0a78ba659..6edb869f43 100644 --- a/packages/agent/vitest.unit.config.ts +++ b/packages/agent/vitest.unit.config.ts @@ -12,6 +12,16 @@ const SQLITE_EXEC_ARGV = [ "--no-warnings=ExperimentalWarning", ]; +// agent.part-16 pins the deterministic post-catchup finalized-SWM drain, but +// it uses the shared chain fixture. Start Hardhat when the full unit inventory +// or that file is selected. Targeted pure-unit jobs (notably the RFC-64 +// Windows gate) keep their existing fast, chain-free path. +const explicitTestFilters = process.argv.filter((arg) => + /(?:^|[/\\])test[/\\]/.test(arg), +); +const needsAgentPart16Fixture = explicitTestFilters.length === 0 + || explicitTestFilters.some((arg) => arg.includes("agent.part-16.test.ts")); + export default defineConfig({ test: { include: [ @@ -102,6 +112,8 @@ export default defineConfig({ "test/finalization-recovery-sqlite-store.test.ts", "test/named-ka-publish-recovery.test.ts", "test/ka-graph-finalization-handler.test.ts", + "test/agent.part-16.test.ts", + "test/sync-responder-swm-subgraphs.test.ts", "test/swm-slice-ka-bound.test.ts", "test/ka-lifecycle-asset-ual-timeout.test.ts", "test/storage-ack-lifecycle-identity.test.ts", @@ -132,6 +144,10 @@ export default defineConfig({ "test/replace-subject-agent-wrapper.test.ts", ], testTimeout: 60_000, + globalSetup: needsAgentPart16Fixture + ? ["../chain/test/hardhat-global-setup.ts"] + : undefined, + env: needsAgentPart16Fixture ? { HARDHAT_PORT: "9545" } : undefined, maxWorkers: 1, pool: "forks", execArgv: SQLITE_EXEC_ARGV, From e84d46156febdb64b14a94dfc1c0021d1aded8fd Mon Sep 17 00:00:00 2001 From: Bojan Date: Fri, 31 Jul 2026 12:39:58 +0200 Subject: [PATCH 13/48] refactor(agent): defer finalized SWM cleanup to idle GC --- packages/agent/src/dkg-agent-base.ts | 3 + packages/agent/src/dkg-agent-constants.ts | 6 + packages/agent/src/dkg-agent-lifecycle.ts | 241 +++++--- packages/agent/src/dkg-agent-swm-substrate.ts | 1 + packages/agent/src/dkg-agent.ts | 8 + packages/agent/src/finalization-handler.ts | 568 +++++++++++------- .../agent/src/finalized-swm-cleanup-marker.ts | 95 +++ .../agent/src/finalized-swm-cleanup-worker.ts | 174 ++++++ .../src/sync/requester/shared-memory-sync.ts | 21 +- .../requester/swm-snapshot-materializer.ts | 287 +++------ .../agent/src/sync/responder/graph-plan.ts | 130 ++-- packages/agent/test/agent.part-16.test.ts | 18 +- .../test/finalized-swm-cleanup-worker.test.ts | 109 ++++ .../ka-graph-finalization-handler.test.ts | 100 ++- ...wm-public-snapshot-materialization.test.ts | 28 +- .../test/swm-snapshot-materializer.test.ts | 200 +++--- .../agent/test/swm-ttl-v2-cleanup.test.ts | 32 +- .../sync-responder-swm-meta-ceiling.test.ts | 13 +- .../test/sync-responder-swm-subgraphs.test.ts | 98 +-- packages/agent/test/workspace-ttl.test.ts | 27 +- packages/agent/vitest.unit.config.ts | 23 +- packages/cli/src/daemon/routes/agent-chat.ts | 20 + packages/cli/test/api-slo-route.test.ts | 39 ++ 23 files changed, 1343 insertions(+), 898 deletions(-) create mode 100644 packages/agent/src/finalized-swm-cleanup-marker.ts create mode 100644 packages/agent/src/finalized-swm-cleanup-worker.ts create mode 100644 packages/agent/test/finalized-swm-cleanup-worker.test.ts diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index 83ab709584..7f91570c91 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -159,6 +159,7 @@ import { type SignedAgentDelegation, } from './auth/agent-delegation.js'; import { SyncVerifyWorker } from './sync-verify-worker.js'; +import type { FinalizedSwmCleanupWorker } from './finalized-swm-cleanup-worker.js'; import { bindRandomSampling, type RandomSamplingDisabledReason, type RandomSamplingHandle, type RandomSamplingStatus } from './random-sampling-bind.js'; import { connectToMultiaddr, ensurePeerConnected as ensurePeerConnectedAtom, primeCatchupConnections as primeCatchupConnectionsAtom } from './p2p/peer-connect.js'; import { Messenger, type SloProtocolStats } from './p2p/messenger.js'; @@ -964,6 +965,8 @@ export class DKGAgentBase { protected messageHandler: MessageHandler | null = null; protected chainPoller: ChainEventPoller | null = null; protected swmCleanupTimer: ReturnType | null = null; + protected finalizedSwmCleanupTimer: ReturnType | null = null; + protected finalizedSwmCleanupWorker?: FinalizedSwmCleanupWorker; /** Phase B — periodic chain-driven VM reconciliation sweep timer. */ protected vmReconcileTimer: ReturnType | null = null; /** Phase B — unified per-CG coalescing and node-wide admission policy. */ diff --git a/packages/agent/src/dkg-agent-constants.ts b/packages/agent/src/dkg-agent-constants.ts index b2fc60750a..6d23bbdcbe 100644 --- a/packages/agent/src/dkg-agent-constants.ts +++ b/packages/agent/src/dkg-agent-constants.ts @@ -136,6 +136,12 @@ export const DEFAULT_SWM_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days export const SWM_CLEANUP_INTERVAL_MS = 15 * 60 * 1000; // run cleanup every 15 minutes export const FINALIZED_SWM_CLEANUP_ROOT_PREDICATE = 'http://dkg.io/ontology/finalizedSwmCleanupRoot'; +export const FINALIZED_SWM_CLEANUP_TASK_TYPE = + 'http://dkg.io/ontology/FinalizedSwmCleanupTask'; +export const FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE = + 'http://dkg.io/ontology/finalizedSwmCleanupMarkedAt'; +export const FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE = + 'http://dkg.io/ontology/finalizedSwmCleanupHeadFingerprint'; export const SYNC_DENIED_RESPONSE = '__DKG_SYNC_DENIED__'; // ── Gossip reconnect ────────────────────────────────────────────────── diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 608ca0a5b0..4ac60b8526 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -338,6 +338,11 @@ import { } from './agent-keystore.js'; import { GossipPublishHandler } from './gossip-publish-handler.js'; import { FinalizationHandler, KEEP_ROOT_COPY_PREDICATE } from './finalization-handler.js'; +import { + FinalizedSwmCleanupWorker, + type FinalizedSwmCleanupStats, + type FinalizedSwmCleanupSweepResult, +} from './finalized-swm-cleanup-worker.js'; import { reconcileContextGraph, RecentUalSet, type ChainReconcilerDeps, type OrdinalOutcome } from './chain-reconciler.js'; import { createCursorState, type CursorState } from './reconcile-cursor.js'; import { resolveStorageAckLifecycleAssetUalFromLocalSwm } from './storage-ack-lifecycle-identity.js'; @@ -3134,13 +3139,20 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); } - // Start periodic SWM maintenance. TTL expiry may be disabled, but exact - // finalized graph-scoped copies still need bounded idle cleanup. - this.cleanupExpiredSharedMemory().catch(() => {}); - this.swmCleanupTimer = setInterval(() => { + // TTL expiry and finalized-SWM GC are deliberately independent. The GC + // worker is pressure-gated and never joined by foreground sync/finalize. + if ((this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS) > 0) { this.cleanupExpiredSharedMemory().catch(() => {}); + this.swmCleanupTimer = setInterval(() => { + this.cleanupExpiredSharedMemory().catch(() => {}); + }, SWM_CLEANUP_INTERVAL_MS); + this.swmCleanupTimer.unref?.(); + } + this.wakeFinalizedSwmCleanup(); + this.finalizedSwmCleanupTimer = setInterval(() => { + this.wakeFinalizedSwmCleanup(); }, SWM_CLEANUP_INTERVAL_MS); - if (this.swmCleanupTimer.unref) this.swmCleanupTimer.unref(); + this.finalizedSwmCleanupTimer.unref?.(); // OT-RFC-38 LU-6: periodic reconciler that ensures the local // node is subscribed in host-mode to every locally-known @@ -6057,21 +6069,10 @@ export class LifecycleSyncMethods extends DKGAgentBase { verifiedPrivateOnlyResponses: cleanDurablePrivateOnlyCompletions, }); } - if (includeSharedMemory && typeof this.cleanupExpiredSharedMemory === 'function') { - // Every selected peer has settled, so this is the first single, - // deterministic cleanup boundary for the whole catch-up job. The - // cleanup's per-KA lock/exact VM+SWM+marker re-checks preserve a - // concurrent newer SWM lifecycle. Queue this bounded deterministic drain - // in the normal maintenance lane: ACK/health reservations remain - // protected, but a continuously deep background sync queue cannot make - // every cleanup attempt expire before admission. The periodic timer - // remains idle-gated/background and is the restart/failure backstop. - await this.cleanupExpiredSharedMemory({ - finalizedOnly: true, - contextGraphIds: [contextGraphId], - finalizedCleanupBudget: 64, - queueBehindActiveWork: true, - }); + if (includeSharedMemory) { + // Catch-up may nudge eventual cleanup but never waits for discovery, + // payload verification or deletion to complete. + this.wakeFinalizedSwmCleanup(); } return { @@ -7592,7 +7593,10 @@ export class LifecycleSyncMethods extends DKGAgentBase { setSharedMemoryTtlMs(this: DKGAgent, ttlMs: number): void { (this.config as any).sharedMemoryTtlMs = ttlMs; - if (!this.swmCleanupTimer) { + if (ttlMs <= 0 && this.swmCleanupTimer) { + clearInterval(this.swmCleanupTimer); + this.swmCleanupTimer = null; + } else if (ttlMs > 0 && !this.swmCleanupTimer) { this.cleanupExpiredSharedMemory().catch(() => {}); this.swmCleanupTimer = setInterval(() => { this.cleanupExpiredSharedMemory().catch(() => {}); @@ -7601,51 +7605,145 @@ export class LifecycleSyncMethods extends DKGAgentBase { } } + getOrCreateFinalizedSwmCleanupWorker(this: DKGAgent): FinalizedSwmCleanupWorker { + if (!this.finalizedSwmCleanupWorker) { + this.finalizedSwmCleanupWorker = new FinalizedSwmCleanupWorker({ + sweep: () => this.runFinalizedSwmCleanupSweep(), + retryDelayMs: 5_000, + onError: (error) => { + this.log.warn( + createOperationContext('system'), + `Finalized SWM cleanup worker failed: ${error instanceof Error ? error.message : String(error)}`, + ); + }, + }); + } + return this.finalizedSwmCleanupWorker; + } + + /** Nudge independent finalized-SWM GC without joining its store work. */ + wakeFinalizedSwmCleanup(this: DKGAgent): void { + this.getOrCreateFinalizedSwmCleanupWorker().wake(); + } + + getFinalizedSwmCleanupStats(this: DKGAgent): FinalizedSwmCleanupStats { + return this.getOrCreateFinalizedSwmCleanupWorker().snapshot(); + } + /** - * Remove expired shared-memory operations and, at safe low-pressure - * boundaries, exact finalized SWM lifecycles retained for deferred cleanup. - * For stale operations it deletes the corresponding triples from shared - * memory and SWM meta, and removes the root entities from - * workspaceOwnedEntities. + * One small idle-only GC slice. Pressure is checked before every discovery + * boundary and between candidates; foreground ACK, health and normal work + * always wins. */ - async cleanupExpiredSharedMemory(this: DKGAgent, options?: { - finalizedOnly?: boolean; - contextGraphIds?: readonly string[]; - finalizedCleanupBudget?: number; - queueBehindActiveWork?: boolean; - }): Promise { + async runFinalizedSwmCleanupSweep(this: DKGAgent): Promise { + const prior = this.finalizedSwmCleanupWorker?.snapshot(); + const priorOldest = prior?.oldestMarkerAgeMs == null + ? null + : Date.now() - prior.oldestMarkerAgeMs; + const pressureResult = (): FinalizedSwmCleanupSweepResult => ({ + backlogDepth: prior?.backlogDepth ?? 0, + oldestMarkerAt: priorOldest, + deletedItems: 0, + pressureSkipped: true, + }); + const budgetResult = ( + backlogDepth: number, + oldestMarkerAt: number | null, + deletedItems: number, + ): FinalizedSwmCleanupSweepResult => ({ + backlogDepth: Math.max(backlogDepth, prior?.backlogDepth ?? 0), + oldestMarkerAt: oldestMarkerAt ?? priorOldest, + deletedItems, + pressureSkipped: false, + budgetExhausted: true, + }); + const underPressure = () => hasActiveStorePressure(this.store.getPressureSnapshot?.()); + if (underPressure()) return pressureResult(); + + const deadline = Date.now() + 10_000; + const deadlineSignal = AbortSignal.timeout(10_000); + let remaining = 4; + let deletedItems = 0; + let backlogDepth = 0; + let oldestMarkerAt: number | null = null; + // Do not even enumerate CGs while the store is busy. + if (underPressure()) return pressureResult(); + const contextGraphs = (await this.listContextGraphs()).map((row) => row.id); + for (const contextGraphId of contextGraphs) { + if (underPressure()) return { ...pressureResult(), deletedItems }; + if (Date.now() >= deadline) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + // Per-CG SWM meta discovery is also forbidden under pressure. + const metaGraphs = await listSharedMemoryMetaGraphs(this.store, contextGraphId); + for (const swmMetaGraph of metaGraphs) { + if (underPressure()) return { ...pressureResult(), deletedItems }; + if (Date.now() >= deadline) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + if (remaining > 0) { + let cleaned: number; + try { + cleaned = await this.getOrCreateFinalizationHandler() + .cleanupFinalizedGraphScopedSwmWhenIdle({ + contextGraphId, + swmMetaGraph, + maxCandidates: 1, + signal: deadlineSignal, + }); + } catch (error) { + if (deadlineSignal.aborted) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + throw error; + } + deletedItems += cleaned; + remaining -= 1; + } + if (underPressure()) return { ...pressureResult(), deletedItems }; + if (Date.now() >= deadline) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + let backlog: { depth: number; oldestMarkerAt: number | null }; + try { + backlog = await this.getOrCreateFinalizationHandler() + .inspectFinalizedGraphScopedSwmCleanupBacklog({ + swmMetaGraph, + signal: deadlineSignal, + }); + } catch (error) { + if (deadlineSignal.aborted) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + throw error; + } + backlogDepth += backlog.depth; + if ( + backlog.oldestMarkerAt !== null + && (oldestMarkerAt === null || backlog.oldestMarkerAt < oldestMarkerAt) + ) { + oldestMarkerAt = backlog.oldestMarkerAt; + } + await new Promise((resolve) => setImmediate(resolve)); + } + } + return { backlogDepth, oldestMarkerAt, deletedItems, pressureSkipped: false }; + } + + /** + * Remove expired shared-memory operations. Finalized-SWM lifecycle GC is + * owned exclusively by FinalizedSwmCleanupWorker above. + */ + async cleanupExpiredSharedMemory(this: DKGAgent): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; + if (ttl <= 0) return 0; const ctx = createOperationContext('share'); - // TTL-disabled nodes still need bounded finalized-SWM maintenance, but an - // ordinary periodic tick must not start graph discovery while foreground - // store work is active. Explicit callers that name CGs or deliberately - // queue behind active work retain deterministic cleanup semantics. - if ( - ttl <= 0 - && !options?.contextGraphIds - && !options?.queueBehindActiveWork - && hasActiveStorePressure(this.store.getPressureSnapshot?.()) - ) { - return 0; - } - const cutoff = !options?.finalizedOnly && ttl > 0 - ? new Date(Date.now() - ttl).toISOString() - : undefined; + const cutoff = new Date(Date.now() - ttl).toISOString(); let totalDeleted = 0; - let finalizedCleanupBudget = Math.max( - 0, - Math.floor(options?.finalizedCleanupBudget ?? 4), - ); try { const graphManager = new GraphManager(this.store); - // A deterministic caller already knows the exact CG IDs. Do not route - // those IDs through GraphManager.listContextGraphs(): that storage-level - // helper intentionally omits owner/name public IDs because it only - // recognizes legacy flat graph IDs. - const contextGraphs = options?.contextGraphIds - ? [...new Set(options.contextGraphIds)] - : await graphManager.listContextGraphs(); + const contextGraphs = await graphManager.listContextGraphs(); for (const pid of contextGraphs) { let graphDeleted = 0; @@ -7661,33 +7759,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { // Each meta graph describes exactly one SWM data bucket: // `…/_shared_memory_meta` ↔ `…/_shared_memory` (root or per-subgraph). const wsGraph = wsMetaGraph.slice(0, -'_meta'.length); - if (finalizedCleanupBudget > 0) { - try { - while (finalizedCleanupBudget > 0) { - const batchSize = Math.min(4, finalizedCleanupBudget); - const cleaned = await this.getOrCreateFinalizationHandler() - .cleanupFinalizedGraphScopedSwmWhenIdle({ - contextGraphId: pid, - swmMetaGraph: wsMetaGraph, - maxCandidates: batchSize, - queueBehindActiveWork: options?.queueBehindActiveWork, - }); - finalizedCleanupBudget -= cleaned; - if (cleaned < batchSize) break; - // Yield between bounded batches so queued foreground work can - // be admitted ahead of the next background-priority batch. - await new Promise((resolve) => setImmediate(resolve)); - } - } catch (error) { - this.log.warn( - ctx, - `Deferred finalized-SWM cleanup failed for ${wsMetaGraph}: ` - + `${error instanceof Error ? error.message : String(error)}`, - ); - } - } - if (!cutoff) continue; - const expiredOps = await this.store.query( `SELECT ?op WHERE { GRAPH <${wsMetaGraph}> { diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 9646b1276c..d42ea81e54 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -1656,6 +1656,7 @@ export class SwmSubstrateMethods extends DKGAgentBase { }, writeLocks: this.writeLocks, publicSnapshotStore: this.publicSnapshotStore, + wakeFinalizedSwmCleanup: () => this.wakeFinalizedSwmCleanup(), runtime: this.finalizationRuntime, }, ); diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 97e763d975..4cf9d7b195 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -1632,6 +1632,14 @@ export class DKGAgent extends DKGAgentBase { clearInterval(this.swmCleanupTimer); this.swmCleanupTimer = null; } + if (this.finalizedSwmCleanupTimer) { + clearInterval(this.finalizedSwmCleanupTimer); + this.finalizedSwmCleanupTimer = null; + } + if (this.finalizedSwmCleanupWorker) { + await this.finalizedSwmCleanupWorker.close(); + this.finalizedSwmCleanupWorker = undefined; + } if (this.hostModeReconcilerTimer) { clearInterval(this.hostModeReconcilerTimer); this.hostModeReconcilerTimer = null; diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index 4c5783435f..3f90107fdb 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -28,6 +28,7 @@ import { type GraphWriteGenSource, type QueryOptions, type SharedMemoryResultBudget, + type StorePressureSnapshot, type SwmKaGraphBound, type TripleStore, type Quad, @@ -62,8 +63,19 @@ import { } from '@origintrail-official/dkg-publisher'; const DKG_NS = 'http://dkg.io/ontology/'; const PROV_NS = 'http://www.w3.org/ns/prov#'; -const FINALIZED_SWM_CLEANUP_BUSY_RETRY_BUDGET_MS = 120_000; -const FINALIZED_SWM_CLEANUP_BUSY_RETRY_DELAY_MS = 250; +function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): boolean { + if (!snapshot) return false; + return [ + snapshot.ackInflight, + snapshot.healthInflight ?? 0, + snapshot.normalInflight, + snapshot.backgroundInflight, + snapshot.ackQueued, + snapshot.healthQueued ?? 0, + snapshot.normalQueued, + snapshot.backgroundQueued, + ].some((count) => count > 0); +} // Slow-query / canary tags for the finalization SWM slice (#1549). A healthy fleet // sees `.fallbackUnbounded` at ~0 relative to `.bounded`; a spike means the bound is @@ -100,8 +112,22 @@ import { type VerifiedGraphScopedFinalizationEvidence, } from './finalization-graph-envelope.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; -import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from './dkg-agent-constants.js'; -export { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from './dkg-agent-constants.js'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; +export { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; +import { + buildFinalizedSwmCleanupTaskQuads, + finalizedSwmCleanupHeadFingerprint, +} from './finalized-swm-cleanup-marker.js'; /** * Predicate for the durable per-root keep-root-copy signal the publisher @@ -303,6 +329,7 @@ export interface FinalizationHandlerOptions { markContextGraphMetaDirtyFromQuads?: MarkContextGraphMetaDirtyFromQuads; writeLocks?: Map>; publicSnapshotStore?: WorkspacePublicSnapshotStore; + wakeFinalizedSwmCleanup?: () => void; lifecycleLogOptions?: FinalizationLifecycleLogOptions; recoveryStore?: FinalizationRecoveryStore; runtime?: FinalizationRuntime; @@ -374,6 +401,7 @@ export class FinalizationHandler { private readonly markContextGraphMetaDirtyFromQuads: MarkContextGraphMetaDirtyFromQuads | undefined; private readonly writeLocks: Map> | undefined; private readonly publicSnapshotStore: WorkspacePublicSnapshotStore | undefined; + private readonly wakeFinalizedSwmCleanup: (() => void) | undefined; private readonly recovery: FinalizationRecovery; private readonly log = new Logger('FinalizationHandler'); private readonly lifecycle: FinalizationLifecycleLogger; @@ -434,6 +462,7 @@ export class FinalizationHandler { this.markContextGraphMetaDirtyFromQuads = options.markContextGraphMetaDirtyFromQuads; this.writeLocks = options.writeLocks; this.publicSnapshotStore = options.publicSnapshotStore; + this.wakeFinalizedSwmCleanup = options.wakeFinalizedSwmCleanup; this.lifecycle = new FinalizationLifecycleLogger( this.log, options.runtime ?? options.lifecycleLogOptions, @@ -1523,14 +1552,15 @@ export class FinalizationHandler { } /** - * Mark one exact, already-materialized SWM head for deferred cleanup. + * Persist one constant-size cleanup task plus an immutable operation + * tombstone. This is the ONLY finalized-SWM work performed in the foreground: + * no payload read, graph hash, cleanup discovery, or deletion is awaited. * - * The finalization path performs only this constant-size metadata write. - * Payload verification and deletion wait for the periodic maintenance lane, - * where store pressure is idle. A newer writer replaces the complete head - * subject under this same lock, which also removes the active-head marker. - * The immutable operation keeps the same marker until normal TTL retention - * removes it, so the finalized recovery snapshot cannot be re-advertised. + * The task subject is independent from the mutable workspace head, so a late + * snapshot replacement cannot erase the worker's durable backlog entry. The + * operation tombstone survives task retirement so a late snapshot can re-arm + * the independent task before it materializes. Ingest still performs no + * discovery, graph verification, or deletion. */ private async markFinalizedGraphScopedSwmForCleanup(input: { contextGraphId: string; @@ -1540,178 +1570,152 @@ export class FinalizationHandler { subGraphName?: string; ctx: OperationContext; }): Promise<'marked' | 'preserved'> { - if (!this.writeLocks) { - this.log.debug( - input.ctx, - `Finalization: preserving graph-scoped SWM for ${input.scope.ual}; ` - + 'no shared SWM writer lock was provided', - ); - return 'preserved'; - } - const lockKey = swmKaWriteLockKey( + const graphManager = new GraphManager(this.store); + const metaGraph = graphManager.sharedMemoryMetaUri( input.contextGraphId, input.subGraphName, - input.scope.ual, ); - return withKeyedLocks(this.writeLocks, [lockKey], async () => { - const graphManager = new GraphManager(this.store); - let currentHead: KnowledgeAssetWorkspaceHead | undefined; - try { - currentHead = await resolveKnowledgeAssetWorkspaceHead({ - store: this.store, - graphManager, - contextGraphId: input.contextGraphId, - kaUal: input.scope.ual, - subGraphName: input.subGraphName, - }); - } catch (error) { - if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; - this.log.warn( - input.ctx, - `Finalization: preserving graph-scoped SWM for ${input.scope.ual}; ` - + `the current workspace head is corrupt: ${error.message}`, - ); - return 'preserved' as const; - } - if (!currentHead || !sameKnowledgeAssetWorkspaceHead(currentHead, input.expectedHead)) { - this.log.info( - input.ctx, - `Finalization: preserving newer graph-scoped SWM lifecycle for ${input.scope.ual}`, - ); - return 'preserved' as const; - } - const metaGraph = graphManager.sharedMemoryMetaUri( - input.contextGraphId, - input.subGraphName, - ); - const cleanupRoot = JSON.stringify( - ethers.hexlify(input.expectedMerkleRoot).toLowerCase(), + const operationSubject = workspaceOperationSubject( + input.contextGraphId, + input.expectedHead.shareOperationId, + ); + const cleanupRootHex = ethers.hexlify(input.expectedMerkleRoot).toLowerCase(); + const cleanupRoot = JSON.stringify(cleanupRootHex); + const markedAtIso = new Date().toISOString(); + const markedAt = `"${markedAtIso}"^^`; + await this.store.insert([ + ...buildFinalizedSwmCleanupTaskQuads({ + contextGraphId: input.contextGraphId, + subGraphName: input.subGraphName, + head: input.expectedHead, + expectedMerkleRootHex: cleanupRootHex, + metaGraph, + markedAtIso, + }), + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: cleanupRoot, + graph: metaGraph, + }, + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: markedAt, + graph: metaGraph, + }, + ]); + this.wakeFinalizedSwmCleanup?.(); + return 'marked'; + } + + /** Read operator backlog gauges without loading any payload graph. */ + async inspectFinalizedGraphScopedSwmCleanupBacklog(input: { + swmMetaGraph: string; + signal?: AbortSignal; + }): Promise<{ depth: number; oldestMarkerAt: number | null }> { + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + throw new StoreSchedulerBusyError( + 'queue_full', + 'background', + 'agent.finalization.graphScopedSwmCleanup.backlog', ); - await this.store.insert([ - { - subject: workspaceKnowledgeAssetHeadSubject(input.scope.ual), - predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, - object: cleanupRoot, - graph: metaGraph, - }, - { - subject: workspaceOperationSubject( - input.contextGraphId, - input.expectedHead.shareOperationId, - ), - predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, - object: cleanupRoot, - graph: metaGraph, - }, - ]); - return 'marked' as const; - }); + } + const result = await this.store.query( + `SELECT (COUNT(DISTINCT ?task) AS ?count) (MIN(?markedAt) AS ?oldest) WHERE { + GRAPH <${assertSafeIri(input.swmMetaGraph)}> { + ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; + <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . + OPTIONAL { ?task <${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } + } + }`, + { + priority: 'background', + source: 'agent.finalization.graphScopedSwmCleanup.backlog', + signal: input.signal, + }, + ); + if (result.type !== 'bindings' || result.bindings.length === 0) { + return { depth: 0, oldestMarkerAt: null }; + } + const depth = Number.parseInt(stripOptionalLiteral(result.bindings[0]?.['count']) ?? '0', 10); + const oldestValue = stripOptionalLiteral(result.bindings[0]?.['oldest']); + const oldestMarkerAt = oldestValue ? Date.parse(oldestValue) : Number.NaN; + return { + depth: Number.isSafeInteger(depth) && depth > 0 ? depth : 0, + oldestMarkerAt: Number.isFinite(oldestMarkerAt) ? oldestMarkerAt : null, + }; } /** * Drain a bounded number of durable finalized-SWM markers only while the - * store scheduler reports no queued or in-flight work. The deterministic - * post-catch-up boundary may explicitly queue cleanup behind active work; - * periodic maintenance remains fully idle-only. + * store scheduler reports no queued or in-flight work. * - * The marker survives restart, while replacing the SWM head for a newer - * assertion removes it automatically. All maintenance queries run in the - * background lane and the destructive step re-enters the canonical per-KA - * writer lock before checking the head, VM, SWM, and marker again. + * The marker survives restart. A newer head causes the worker to retire the + * obsolete task without touching that newer lifecycle. All maintenance + * queries run in the background lane and the destructive step re-enters the + * canonical per-KA writer lock only for the final head/task re-read and + * conditional delete. */ async cleanupFinalizedGraphScopedSwmWhenIdle(input: { contextGraphId: string; swmMetaGraph: string; maxCandidates?: number; - queueBehindActiveWork?: boolean; + signal?: AbortSignal; }): Promise { if (!this.writeLocks) return 0; const pressure = this.store.getPressureSnapshot?.(); - if (!input.queueBehindActiveWork && pressure && ( - pressure.ackInflight > 0 - || (pressure.healthInflight ?? 0) > 0 - || pressure.normalInflight > 0 - || pressure.backgroundInflight > 0 - || pressure.ackQueued > 0 - || (pressure.healthQueued ?? 0) > 0 - || pressure.normalQueued > 0 - || pressure.backgroundQueued > 0 - )) { + if (hasActiveStorePressure(pressure)) { return 0; } const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); - // A deterministic post-catch-up drain is part of completing the lifecycle - // job, not opportunistic maintenance. Give it the normal lane so a - // continuously deep background sync queue cannot make every retry expire - // before admission. ACK and health work retain their reserved slots, while - // periodic cleanup stays background-only and idle-gated. const cleanupQueryOptions: QueryOptions = { - priority: input.queueBehindActiveWork ? 'normal' : 'background', + priority: 'background', source: 'agent.finalization.graphScopedSwmCleanup.discover', + signal: input.signal, }; - const busyRetryDeadline = input.queueBehindActiveWork - ? Date.now() + FINALIZED_SWM_CLEANUP_BUSY_RETRY_BUDGET_MS - : 0; - const runStoreOperation = async (operation: () => Promise): Promise => { - for (;;) { - try { - return await operation(); - } catch (error) { - if ( - !(error instanceof StoreSchedulerBusyError) - || !input.queueBehindActiveWork - || Date.now() >= busyRetryDeadline - ) { - throw error; - } - await new Promise((resolve) => { - setTimeout(resolve, Math.min( - FINALIZED_SWM_CLEANUP_BUSY_RETRY_DELAY_MS, - Math.max(1, busyRetryDeadline - Date.now()), - )); - }); - } - } - }; - const result = await runStoreOperation(() => this.store.query( - `SELECT DISTINCT ?head ?ual ?version ?root ?shareId ?subGraphName WHERE { + const result = await this.store.query( + `SELECT DISTINCT ?task ?ual ?version ?root ?shareId ?assertionGraph ?headFingerprint ?subGraphName WHERE { GRAPH <${assertSafeIri(input.swmMetaGraph)}> { - ?head <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; + ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; + <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; + <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ?headFingerprint ; <${DKG_NS}kaUal> ?ual ; <${DKG_NS}assertionVersion> ?version ; <${DKG_NS}shareOperationId> ?shareId ; <${DKG_NS}assertionGraph> ?assertionGraph . - ?operation <${DKG_NS}WorkspaceOperation> ; - <${DKG_NS}shareOperationId> ?shareId ; - <${DKG_NS}kaUal> ?ual ; - <${DKG_NS}assertionVersion> ?version . - OPTIONAL { ?operation <${DKG_NS}subGraphName> ?subGraphName } + OPTIONAL { ?task <${DKG_NS}subGraphName> ?subGraphName } } - } ORDER BY ?head LIMIT ${limit}`, + } ORDER BY ?task LIMIT ${limit}`, cleanupQueryOptions, - )); + ); if (result.type !== 'bindings') return 0; let cleared = 0; for (const row of result.bindings) { const currentPressure = this.store.getPressureSnapshot?.(); - if (!input.queueBehindActiveWork && currentPressure && ( - currentPressure.ackInflight > 0 - || (currentPressure.healthInflight ?? 0) > 0 - || currentPressure.normalInflight > 0 - || currentPressure.backgroundInflight > 0 - || currentPressure.ackQueued > 0 - || (currentPressure.healthQueued ?? 0) > 0 - || currentPressure.normalQueued > 0 - || currentPressure.backgroundQueued > 0 - )) { + if (hasActiveStorePressure(currentPressure)) { break; } const ual = row['ual']; + const taskSubject = row['task']; const rawVersion = stripOptionalLiteral(row['version']); const rawRoot = stripOptionalLiteral(row['root']); + const assertionGraph = row['assertionGraph']; + const expectedHeadFingerprint = stripOptionalLiteral(row['headFingerprint']); + const shareOperationId = stripOptionalLiteral(row['shareId']); const subGraphName = stripOptionalLiteral(row['subGraphName']); - if (!ual || !rawVersion || !/^\d+$/.test(rawVersion) || !rawRoot) continue; + if ( + !taskSubject + || !taskSubject.startsWith('urn:dkg:finalized-swm-cleanup:') + || !ual + || !rawVersion + || !/^\d+$/.test(rawVersion) + || !rawRoot + || !assertionGraph + || !expectedHeadFingerprint + || !shareOperationId + ) continue; let scope: ReturnType; let expectedMerkleRoot: Uint8Array; try { @@ -1723,33 +1727,54 @@ export class FinalizationHandler { if ( scope.ual !== ual || expectedMerkleRoot.length !== 32 - || workspaceKnowledgeAssetHeadSubject(ual) !== row['head'] ) { continue; } const graphManager = new GraphManager(this.store); let expectedHead: KnowledgeAssetWorkspaceHead | undefined; try { - expectedHead = await runStoreOperation(() => resolveKnowledgeAssetWorkspaceHead({ + expectedHead = await resolveKnowledgeAssetWorkspaceHead({ store: this.store, graphManager, contextGraphId: input.contextGraphId, kaUal: scope.ual, subGraphName, queryOptions: cleanupQueryOptions, - })); + }); } catch (error) { if (error instanceof StoreSchedulerBusyError) { - if (input.queueBehindActiveWork) throw error; break; } continue; } + if (!expectedHead) { + await this.retireStaleFinalizedSwmCleanupTask({ + contextGraphId: input.contextGraphId, + swmMetaGraph: input.swmMetaGraph, + taskSubject, + scope, + assertionGraph, + shareOperationId, + subGraphName, + queryOptions: cleanupQueryOptions, + }); + continue; + } if ( - !expectedHead - || expectedHead.assertionVersion !== scope.assertionVersion - || expectedHead.shareOperationId !== stripOptionalLiteral(row['shareId']) + expectedHead.assertionVersion !== scope.assertionVersion + || expectedHead.shareOperationId !== shareOperationId + || expectedHead.assertionGraph !== assertionGraph ) { + await this.retireStaleFinalizedSwmCleanupTask({ + contextGraphId: input.contextGraphId, + swmMetaGraph: input.swmMetaGraph, + taskSubject, + scope, + assertionGraph, + shareOperationId, + subGraphName, + queryOptions: cleanupQueryOptions, + }); continue; } let privateMerkleRoot: Uint8Array | undefined; @@ -1760,25 +1785,98 @@ export class FinalizationHandler { } catch { continue; } - const outcome = await runStoreOperation(() => this.clearMarkedFinalizedGraphScopedSwm({ + const outcome = await this.clearMarkedFinalizedGraphScopedSwm({ contextGraphId: input.contextGraphId, scope, + taskSubject, + expectedHeadFingerprint, expectedHead, expectedMerkleRoot, privateMerkleRoot, subGraphName, queryPriority: cleanupQueryOptions.priority, ctx: createOperationContext('system'), - })); - if (outcome === 'cleared' || outcome === 'absent') cleared += 1; + }); + if (outcome === 'cleared') cleared += 1; } return cleared; } + /** + * Retire an obsolete task under the same per-KA writer lock used by SWM + * materialization. This is the short lock-held re-read/conditional-delete + * step: no VM graph read, digest, or Merkle work occurs here. + */ + private async retireStaleFinalizedSwmCleanupTask(input: { + contextGraphId: string; + swmMetaGraph: string; + taskSubject: string; + scope: ReturnType; + assertionGraph: string; + shareOperationId: string; + subGraphName?: string; + queryOptions: QueryOptions; + }): Promise { + if (!this.writeLocks) return; + const lockKey = swmKaWriteLockKey( + input.contextGraphId, + input.subGraphName, + input.scope.ual, + ); + await withKeyedLocks(this.writeLocks, [lockKey], async () => { + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager: new GraphManager(this.store), + contextGraphId: input.contextGraphId, + kaUal: input.scope.ual, + subGraphName: input.subGraphName, + queryOptions: input.queryOptions, + }); + } catch (error) { + if (error instanceof KnowledgeAssetWorkspaceHeadCorruptError) return; + throw error; + } + if ( + currentHead + && currentHead.assertionVersion === input.scope.assertionVersion + && currentHead.shareOperationId === input.shareOperationId + && currentHead.assertionGraph === input.assertionGraph + ) { + return; + } + if (!currentHead) { + // An interrupted snapshot metadata replacement can leave payload plus + // task without a head. Preserve the task until the retry restores its + // metadata; retire only when both mutable pieces are absent. + const payloadPresent = await this.store.query( + `ASK { GRAPH <${assertSafeIri(input.assertionGraph)}> { ?s ?p ?o } }`, + { + ...input.queryOptions, + source: 'agent.finalization.graphScopedSwmCleanup.checkHeadlessPayload', + }, + ); + if (payloadPresent.type !== 'boolean' || payloadPresent.value) return; + } + await this.store.deleteByPattern( + { graph: input.swmMetaGraph, subject: input.taskSubject }, + { + ...input.queryOptions, + source: currentHead + ? 'agent.finalization.graphScopedSwmCleanup.retireSupersededTask' + : 'agent.finalization.graphScopedSwmCleanup.retireAbsentTask', + }, + ); + }); + } + /** Atomically remove only the still-marked, still-exact active SWM lifecycle. */ private async clearMarkedFinalizedGraphScopedSwm(input: { contextGraphId: string; scope: ReturnType; + taskSubject: string; + expectedHeadFingerprint: string; expectedHead: KnowledgeAssetWorkspaceHead; expectedMerkleRoot: Uint8Array; privateMerkleRoot?: Uint8Array; @@ -1790,6 +1888,8 @@ export class FinalizationHandler { const { contextGraphId, scope, + taskSubject, + expectedHeadFingerprint, expectedHead, expectedMerkleRoot, privateMerkleRoot, @@ -1797,13 +1897,89 @@ export class FinalizationHandler { queryPriority, ctx, } = input; + const graphManager = new GraphManager(this.store); + const metaGraph = graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); + const headSubject = workspaceKnowledgeAssetHeadSubject(scope.ual); + const cleanupRootObject = JSON.stringify( + ethers.hexlify(expectedMerkleRoot).toLowerCase(), + ); + const cleanupQueryOptions: QueryOptions = { + priority: queryPriority ?? 'background', + source: 'agent.finalization.graphScopedSwmCleanup', + }; + if (finalizedSwmCleanupHeadFingerprint(expectedHead) !== expectedHeadFingerprint) { + this.log.warn(ctx, `Finalization cleanup: preserving ${scope.ual}; cleanup task fingerprint differs`); + return 'preserved'; + } + + // Expensive graph reads and hashing happen before the writer lock. The + // write-generation snapshot proves that no local writer changed any graph + // in this CG between verification and the final lock-held commit. + const writePrefix = `${contextGraphDataUri(contextGraphId)}/`; + const preflightWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + if (queryPriority !== 'normal' && hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + return 'preserved'; + } + const vmVerification = await this.verifyExactGraphScopedLayer({ + contextGraphId, + scope, + layer: MemoryLayer.VerifiableMemory, + publicTripleCount: expectedHead.publicTripleCount, + privateMerkleRoot, + expectedMerkleRoot, + expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, + subGraphName, + queryOptions: cleanupQueryOptions, + }); + if (vmVerification.status !== 'verified') { + this.log.warn( + ctx, + `Finalization cleanup: preserving graph-scoped SWM for ${scope.ual}; ` + + `VM no longer matches the cleanup token (${vmVerification.status})`, + ); + return 'preserved'; + } + if (queryPriority !== 'normal' && hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + return 'preserved'; + } + const swmVerification = await this.verifyExactGraphScopedLayer({ + contextGraphId, + scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: expectedHead.publicTripleCount, + privateMerkleRoot, + expectedMerkleRoot, + expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, + subGraphName, + queryOptions: cleanupQueryOptions, + }); + if ( + swmVerification.status !== 'verified' + && !(swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) + ) { + this.log.warn( + ctx, + `Finalization cleanup: preserving graph-scoped SWM for ${scope.ual}; ` + + `the current assertion no longer matches the finalized source (${swmVerification.status})`, + ); + return 'preserved'; + } + const verifiedWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + if ( + preflightWriteGen !== undefined + && verifiedWriteGen !== preflightWriteGen + ) { + return 'preserved'; + } + const lockKey = swmKaWriteLockKey(contextGraphId, subGraphName, scope.ual); const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { - const graphManager = new GraphManager(this.store); - const cleanupQueryOptions: QueryOptions = { - priority: queryPriority ?? 'background', - source: 'agent.finalization.graphScopedSwmCleanup', - }; + if ( + verifiedWriteGen !== undefined + && this.graphWriteGen?.getWriteGen(writePrefix) !== verifiedWriteGen + ) { + return 'preserved' as const; + } let currentHead: KnowledgeAssetWorkspaceHead | undefined; try { currentHead = await resolveKnowledgeAssetWorkspaceHead({ @@ -1823,71 +1999,35 @@ export class FinalizationHandler { ); return 'preserved' as const; } - if (!currentHead) return 'absent' as const; - if (!sameKnowledgeAssetWorkspaceHead(currentHead, expectedHead)) { + if (!currentHead) { + if (swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) { + await this.store.deleteByPattern( + { graph: metaGraph, subject: taskSubject }, + { ...cleanupQueryOptions, source: 'agent.finalization.graphScopedSwmCleanup.retireAbsentTask' }, + ); + return 'absent' as const; + } + return 'preserved' as const; + } + if ( + !sameKnowledgeAssetWorkspaceHead(currentHead, expectedHead) + || finalizedSwmCleanupHeadFingerprint(currentHead) !== expectedHeadFingerprint + ) { this.log.info( ctx, `Finalization: preserving newer graph-scoped SWM lifecycle for ${scope.ual}`, ); return 'preserved' as const; } - - const metaGraph = graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); - const headSubject = workspaceKnowledgeAssetHeadSubject(scope.ual); - const cleanupRootObject = JSON.stringify( - ethers.hexlify(expectedMerkleRoot).toLowerCase(), - ); const marker = await this.store.query( `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` - + `<${assertSafeIri(headSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` - + `${cleanupRootObject} } }`, + + `<${assertSafeIri(taskSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` + + `${cleanupRootObject} ; <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ` + + `${JSON.stringify(expectedHeadFingerprint)} } }`, cleanupQueryOptions, ); if (marker.type !== 'boolean' || !marker.value) return 'preserved' as const; - const vmVerification = await this.verifyExactGraphScopedLayer({ - contextGraphId, - scope, - layer: MemoryLayer.VerifiableMemory, - publicTripleCount: expectedHead.publicTripleCount, - privateMerkleRoot, - expectedMerkleRoot, - expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - if (vmVerification.status !== 'verified') { - this.log.warn( - ctx, - `Finalization cleanup: preserving graph-scoped SWM for ${scope.ual}; ` - + `VM no longer matches the cleanup token (${vmVerification.status})`, - ); - return 'preserved' as const; - } - - const swmVerification = await this.verifyExactGraphScopedLayer({ - contextGraphId, - scope, - layer: MemoryLayer.SharedWorkingMemory, - publicTripleCount: expectedHead.publicTripleCount, - privateMerkleRoot, - expectedMerkleRoot, - expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - if ( - swmVerification.status !== 'verified' - && !(swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) - ) { - this.log.warn( - ctx, - `Finalization: preserving graph-scoped SWM for ${scope.ual}; ` - + `the current assertion no longer matches the finalized source (${swmVerification.status})`, - ); - return 'preserved' as const; - } - const replaced = await tryReplaceGraphAndSubjectAtomically( this.store, swmVerification.graphUri, @@ -1903,10 +2043,16 @@ export class FinalizationHandler { { code: 'SWM_ATOMIC_CLEANUP_UNSUPPORTED' }, ); } + // Delete the independent task last. A crash after the data/head commit + // leaves only a harmless retry; deleting the task first could lose work. + await this.store.deleteByPattern( + { graph: metaGraph, subject: taskSubject }, + { ...cleanupQueryOptions, source: 'agent.finalization.graphScopedSwmCleanup.retireTask' }, + ); return swmVerification.status === 'verified' ? 'cleared' as const : 'absent' as const; }); - if (outcome === 'cleared' || outcome === 'absent') { + if (outcome === 'cleared') { this.eventBus?.emit(DKGEvent.MEMORY_GRAPH_CHANGED, { contextGraphId, layers: ['swm'], diff --git a/packages/agent/src/finalized-swm-cleanup-marker.ts b/packages/agent/src/finalized-swm-cleanup-marker.ts new file mode 100644 index 0000000000..759b9224fe --- /dev/null +++ b/packages/agent/src/finalized-swm-cleanup-marker.ts @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from 'node:crypto'; +import { GRAPH_KA_CONTENT_SCOPE_VERSION } from '@origintrail-official/dkg-core'; +import type { Quad } from '@origintrail-official/dkg-storage'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; + +const DKG = 'http://dkg.io/ontology/'; +const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; +const XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer'; +const XSD_DATE_TIME = 'http://www.w3.org/2001/XMLSchema#dateTime'; + +/** Stable fields that identify one exact graph-scoped SWM lifecycle. */ +export interface FinalizedSwmCleanupHeadIdentity { + kaUal: string; + assertionVersion: bigint | number | string; + assertionGraph: string; + publicQuadsDigest: string; + publicTripleCount: number; + privateMerkleRoot?: string; + privateTripleCount: number; + shareOperationId: string; + publisherPeerId: string; + accessPolicy?: string; + allowedPeers: readonly string[]; +} + +export function finalizedSwmCleanupHeadFingerprint( + head: FinalizedSwmCleanupHeadIdentity, +): string { + return createHash('sha256').update(JSON.stringify({ + kaUal: head.kaUal, + assertionVersion: head.assertionVersion.toString(), + assertionGraph: head.assertionGraph, + publicQuadsDigest: head.publicQuadsDigest, + publicTripleCount: head.publicTripleCount, + privateMerkleRoot: head.privateMerkleRoot?.toLowerCase(), + privateTripleCount: head.privateTripleCount, + shareOperationId: head.shareOperationId, + publisherPeerId: head.publisherPeerId, + accessPolicy: head.accessPolicy, + allowedPeers: [...head.allowedPeers].sort(), + })).digest('hex'); +} + +export function finalizedSwmCleanupTaskSubject(input: { + contextGraphId: string; + subGraphName?: string; + head: FinalizedSwmCleanupHeadIdentity; +}): string { + const key = JSON.stringify([ + input.contextGraphId, + input.subGraphName ?? '', + input.head.kaUal, + input.head.assertionVersion.toString(), + input.head.shareOperationId, + ]); + return `urn:dkg:finalized-swm-cleanup:${createHash('sha256').update(key).digest('hex')}`; +} + +/** Build the durable, constant-size work item owned by the idle GC worker. */ +export function buildFinalizedSwmCleanupTaskQuads(input: { + contextGraphId: string; + subGraphName?: string; + head: FinalizedSwmCleanupHeadIdentity; + expectedMerkleRootHex: string; + metaGraph: string; + markedAtIso: string; +}): Quad[] { + const taskSubject = finalizedSwmCleanupTaskSubject(input); + const integer = (value: bigint | number | string) => + `"${String(value)}"^^<${XSD_INTEGER}>`; + return [ + { subject: taskSubject, predicate: RDF_TYPE, object: FINALIZED_SWM_CLEANUP_TASK_TYPE, graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}contentScopeVersion`, object: integer(GRAPH_KA_CONTENT_SCOPE_VERSION), graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}kaUal`, object: input.head.kaUal, graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}assertionVersion`, object: integer(input.head.assertionVersion), graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}shareOperationId`, object: JSON.stringify(input.head.shareOperationId), graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}assertionGraph`, object: input.head.assertionGraph, graph: input.metaGraph }, + { subject: taskSubject, predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, object: JSON.stringify(input.expectedMerkleRootHex.toLowerCase()), graph: input.metaGraph }, + { subject: taskSubject, predicate: FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, object: JSON.stringify(finalizedSwmCleanupHeadFingerprint(input.head)), graph: input.metaGraph }, + { subject: taskSubject, predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, object: `"${input.markedAtIso}"^^<${XSD_DATE_TIME}>`, graph: input.metaGraph }, + ...(input.subGraphName ? [{ + subject: taskSubject, + predicate: `${DKG}subGraphName`, + object: JSON.stringify(input.subGraphName), + graph: input.metaGraph, + }] : []), + ]; +} diff --git a/packages/agent/src/finalized-swm-cleanup-worker.ts b/packages/agent/src/finalized-swm-cleanup-worker.ts new file mode 100644 index 0000000000..04f4108da7 --- /dev/null +++ b/packages/agent/src/finalized-swm-cleanup-worker.ts @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Independent, pressure-gated maintenance worker for finalized SWM cleanup. + * + * Foreground finalization and catch-up only call {@link wake}; they never await + * discovery, payload verification, or deletion. The worker owns retry and + * single-flight semantics so repeated wake-ups cannot create overlapping store + * scans. A periodic caller remains the restart/lost-wakeup safety net. + */ + +export interface FinalizedSwmCleanupSweepResult { + /** Cleanup tasks still requiring a safe idle pass after this sweep. */ + backlogDepth: number; + /** Oldest surviving cleanup marker, or null when the backlog is empty/unknown. */ + oldestMarkerAt: number | null; + /** Exact SWM lifecycles deleted by this sweep. */ + deletedItems: number; + /** True when pressure appeared before or during discovery/cleanup. */ + pressureSkipped: boolean; + /** True when the bounded slice yielded with more discovery work possible. */ + budgetExhausted?: boolean; +} + +export interface FinalizedSwmCleanupStats { + backlogDepth: number; + oldestMarkerAgeMs: number | null; + pressureSkips: number; + deletedItems: number; + runs: number; + lastRunAt: string | null; + lastError: string | null; +} + +export interface FinalizedSwmCleanupWorkerOptions { + sweep: () => Promise; + now?: () => number; + retryDelayMs?: number; + setTimer?: (fn: () => void, delayMs: number) => ReturnType; + clearTimer?: (timer: ReturnType) => void; + onError?: (error: unknown) => void; +} + +export class FinalizedSwmCleanupWorker { + private readonly sweep: () => Promise; + private readonly now: () => number; + private readonly retryDelayMs: number; + private readonly setTimer: NonNullable; + private readonly clearTimer: NonNullable; + private readonly onError: (error: unknown) => void; + private wakeTimer: ReturnType | null = null; + private inFlight: Promise | null = null; + private wakePendingDelayMs: number | null = null; + private closed = false; + private oldestMarkerAt: number | null = null; + private readonly stats: FinalizedSwmCleanupStats = { + backlogDepth: 0, + oldestMarkerAgeMs: null, + pressureSkips: 0, + deletedItems: 0, + runs: 0, + lastRunAt: null, + lastError: null, + }; + + constructor(options: FinalizedSwmCleanupWorkerOptions) { + this.sweep = options.sweep; + this.now = options.now ?? Date.now; + this.retryDelayMs = Math.max(1, Math.floor(options.retryDelayMs ?? 5_000)); + this.setTimer = options.setTimer ?? ((fn, delayMs) => setTimeout(fn, delayMs)); + this.clearTimer = options.clearTimer ?? clearTimeout; + this.onError = options.onError ?? (() => {}); + } + + /** Schedule an independent maintenance pass; never waits for store work. */ + wake(delayMs = 0): void { + if (this.closed) return; + if (this.inFlight) { + const normalizedDelay = Math.max(0, Math.floor(delayMs)); + this.wakePendingDelayMs = this.wakePendingDelayMs === null + ? normalizedDelay + : Math.min(this.wakePendingDelayMs, normalizedDelay); + return; + } + if (this.wakeTimer) return; + this.wakeTimer = this.setTimer(() => { + this.wakeTimer = null; + void this.run(); + }, Math.max(0, Math.floor(delayMs))); + this.wakeTimer.unref?.(); + } + + /** Test/diagnostic seam that joins the current pass without starting overlap. */ + async runNow(): Promise { + if (this.closed) return; + if (this.wakeTimer) { + this.clearTimer(this.wakeTimer); + this.wakeTimer = null; + } + await this.run(); + } + + snapshot(): FinalizedSwmCleanupStats { + return { + ...this.stats, + oldestMarkerAgeMs: this.oldestMarkerAt === null + ? null + : Math.max(0, this.now() - this.oldestMarkerAt), + }; + } + + async close(): Promise { + this.closed = true; + this.wakePendingDelayMs = null; + if (this.wakeTimer) { + this.clearTimer(this.wakeTimer); + this.wakeTimer = null; + } + await this.inFlight; + } + + private async run(): Promise { + if (this.closed) return; + if (this.inFlight) { + this.wakePendingDelayMs = 0; + await this.inFlight; + return; + } + const task = this.performSweep(); + this.inFlight = task; + try { + await task; + } finally { + if (this.inFlight === task) this.inFlight = null; + if (this.closed) return; + if (this.wakePendingDelayMs !== null) { + const delayMs = this.wakePendingDelayMs; + this.wakePendingDelayMs = null; + this.wake(delayMs); + } + } + } + + private async performSweep(): Promise { + const startedAt = this.now(); + this.stats.runs += 1; + this.stats.lastRunAt = new Date(startedAt).toISOString(); + try { + const result = await this.sweep(); + this.stats.backlogDepth = Math.max(0, Math.floor(result.backlogDepth)); + this.oldestMarkerAt = result.oldestMarkerAt; + this.stats.oldestMarkerAgeMs = this.oldestMarkerAt === null + ? null + : Math.max(0, this.now() - this.oldestMarkerAt); + this.stats.deletedItems += Math.max(0, Math.floor(result.deletedItems)); + if (result.pressureSkipped) this.stats.pressureSkips += 1; + this.stats.lastError = null; + + // Pressure is expected and retryable. Continue a productive bounded + // drain without waiting for the 15-minute periodic backstop; a stuck or + // fail-closed backlog (no deletion) waits for the next explicit wake. + if ( + result.pressureSkipped + || result.budgetExhausted === true + || (result.deletedItems > 0 && result.backlogDepth > 0) + ) { + this.wake(this.retryDelayMs); + } + } catch (error) { + this.stats.lastError = error instanceof Error ? error.message : String(error); + this.onError(error); + } + } +} diff --git a/packages/agent/src/sync/requester/shared-memory-sync.ts b/packages/agent/src/sync/requester/shared-memory-sync.ts index ee5717a53b..2fc92843bf 100644 --- a/packages/agent/src/sync/requester/shared-memory-sync.ts +++ b/packages/agent/src/sync/requester/shared-memory-sync.ts @@ -317,22 +317,11 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // re-checks below stop the other failure the lock alone cannot: // replacing newer content with an older verified snapshot. // - // (0) Finalized anti-resurrection. The deferred cleanup may - // have drained this exact assertion while another peer's SWM - // sync was still in flight. VM is the authoritative proof: - // once this descriptor is exactly confirmed there, never - // restore it to SWM. The materializer also removes an exact - // active duplicate atomically, while preserving a newer or - // otherwise different SWM lifecycle. - if (await snapshotMaterializer.discardFinalizedGraphAsset(pid, descriptor)) { - for (const quad of descriptor.metadataQuads) { - replacedGraphScopedMetaKeys.add(quadKey(quad)); - } - materializedKeys.add(graphKey); - logDebug(ctx, `SWM sync for "${pid}": snapshot ${snapshotRef} is already ` - + 'exactly confirmed in VM; refusing SWM restoration'); - return; - } + // Re-arm only constant-size durable maintenance metadata when + // an earlier finalized cleanup left an operation tombstone. + // The independent GC owns all discovery, graph verification + // and deletion; sync neither performs nor awaits that work. + await snapshotMaterializer.ensureFinalizedCleanupTask(pid, descriptor); // // (a) Version ordering. A stored head newer than the descriptor // means gossip advanced this KA past our snapshot; replacing diff --git a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts index 3abe67e164..f709197086 100644 --- a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts +++ b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts @@ -15,22 +15,20 @@ */ import { assertSafeIri } from '@origintrail-official/dkg-core'; import { - computeFlatKCRootV10, - readConfirmedGraphKnowledgeAssetMetadataEnvelope, - resolveKnowledgeAssetWorkspaceHead, - sameKnowledgeAssetWorkspaceHead, swmKaWriteLockKey, withKeyedLocks, workspacePublicQuadsDigest, } from '@origintrail-official/dkg-publisher'; import { - GraphManager, - tryReplaceGraphAndSubjectAtomically, type Quad, type TripleStore, } from '@origintrail-official/dkg-storage'; import type { GraphScopedSwmRecoveryDescriptor } from '../graph-scoped-swm-recovery.js'; -import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from '../../dkg-agent-constants.js'; +import { + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, +} from '../../dkg-agent-constants.js'; +import { buildFinalizedSwmCleanupTaskQuads } from '../../finalized-swm-cleanup-marker.js'; const DKG = 'http://dkg.io/ontology/'; @@ -95,19 +93,14 @@ export interface SharedMemorySnapshotMaterializer { */ isGraphAssetMaterialized(descriptor: GraphScopedSwmRecoveryDescriptor): Promise; /** - * Refuse to resurrect an assertion that is already exactly confirmed in VM. - * When the active SWM head still names this descriptor, remove its exact - * duplicate graph and head atomically. A newer/different SWM head is - * preserved, but the stale finalized descriptor is still rejected. - * - * Returns true when the incoming descriptor is already exactly finalized, - * whether the local duplicate was removed, already absent, or preserved - * because the active SWM lifecycle differs. + * Re-arm a constant-size GC task when this exact operation carries a local + * finalization tombstone. This is marker bookkeeping only: no graph scan, + * VM verification, deletion, or GC wait occurs on the ingest path. */ - discardFinalizedGraphAsset( + ensureFinalizedCleanupTask( contextGraphId: string, descriptor: GraphScopedSwmRecoveryDescriptor, - ): Promise; + ): Promise; /** * Atomic whole-graph replace. Replace, not insert: a KA graph is * all-or-nothing and digest-verified; union-insert risks partial or @@ -130,18 +123,17 @@ export interface SharedMemorySnapshotMaterializer { } /** - * Replace one graph-scoped SWM lifecycle without losing a local deferred - * finalization token for the exact same lifecycle. + * Replace one graph-scoped SWM lifecycle without losing its immutable local + * finalization tombstone. * * Finalization markers are deliberately local-only and responders filter them * from synchronized metadata. A blind head/operation replacement therefore - * erased the only durable evidence that the retained SWM graph was eligible - * for idle cleanup. Preserve both marker rows only when the complete local - * workspace head still equals the verified incoming descriptor; any mismatch - * means a newer or otherwise different lifecycle and fails closed by dropping - * the old markers. + * erased the metadata-only lifecycle record needed to re-arm background GC. + * Preserve the operation tombstone only for the exact incoming operation + * subject. The independent GC task is stored on its own subject and is not + * touched by this replacement. * - * The verified replacement metadata and any retained cleanup markers are + * The verified replacement metadata and any retained operation tombstone are * inserted in the same store call after the old lifecycle is removed. Callers * MUST hold the canonical per-KA SWM writer lock across this entire operation. */ @@ -159,7 +151,7 @@ export async function replaceGraphScopedSwmHeadMetadata(params: { sourcePrefix, insertReplacementMetadata, } = params; - const preservedMarkers = await readExactFinalizedCleanupMarkers({ + const preserved = await readExactFinalizedOperationTombstone({ store, contextGraphId, descriptor, @@ -204,151 +196,78 @@ export async function replaceGraphScopedSwmHeadMetadata(params: { } const replacementQuads = [ ...descriptor.metadataQuads, - ...preservedMarkers, + ...preserved.tombstone, ]; if (replacementQuads.length > 0) { await insertReplacementMetadata(replacementQuads); } } -async function readExactFinalizedCleanupMarkers(params: { +async function readExactFinalizedOperationTombstone(params: { store: TripleStore; contextGraphId: string; descriptor: GraphScopedSwmRecoveryDescriptor; sourcePrefix: string; -}): Promise { - const { store, contextGraphId, descriptor, sourcePrefix } = params; - try { - const currentHead = await resolveKnowledgeAssetWorkspaceHead({ - store, - graphManager: new GraphManager(store), - contextGraphId, - kaUal: descriptor.kaUal, - subGraphName: descriptor.subGraphName, - queryOptions: { - priority: 'background', - source: `${sourcePrefix}.readFinalizedCleanupHead`, - }, - }); - if (!currentHead || !sameKnowledgeAssetWorkspaceHead( - currentHead, - workspaceHeadFromDescriptor(descriptor), - )) { - return []; - } - } catch { - // Corrupt or incomplete local metadata must never carry a destructive - // cleanup token into a verified replacement. - return []; - } - +}): Promise<{ tombstone: Quad[]; cleanupTask: Quad[] }> { + const { store, descriptor, sourcePrefix } = params; const markerResult = await store.query( - `CONSTRUCT { ?subject <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } WHERE { ` + `CONSTRUCT { <${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . ` + + `<${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } WHERE { ` + `GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` - + `VALUES ?subject { <${assertSafeIri(descriptor.headSubject)}> ` - + `<${assertSafeIri(descriptor.operationSubject)}> } ` - + `?subject <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + + `<${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . ` + + `OPTIONAL { <${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } } }`, { priority: 'background', - source: `${sourcePrefix}.readFinalizedCleanupMarkers`, + source: `${sourcePrefix}.readFinalizedOperationTombstone`, }, ); - if (markerResult.type !== 'quads') return []; + if (markerResult.type !== 'quads') return { tombstone: [], cleanupTask: [] }; const markers = markerResult.quads.map((quad) => ({ ...quad, graph: descriptor.metaGraph, })); - const headMarkers = markers.filter((quad) => quad.subject === descriptor.headSubject); - const operationMarkers = markers.filter((quad) => quad.subject === descriptor.operationSubject); - if ( - headMarkers.length !== 1 - || operationMarkers.length !== 1 - || headMarkers[0]!.object !== operationMarkers[0]!.object - ) { - return []; + const roots = markers.filter((quad) => + quad.subject === descriptor.operationSubject + && quad.predicate === FINALIZED_SWM_CLEANUP_ROOT_PREDICATE); + const markedAtRows = markers.filter((quad) => + quad.subject === descriptor.operationSubject + && quad.predicate === FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE); + if (roots.length !== 1) { + return { tombstone: [], cleanupTask: [] }; } - return [headMarkers[0]!, operationMarkers[0]!]; -} - -function workspaceHeadFromDescriptor( - descriptor: GraphScopedSwmRecoveryDescriptor, -) { + const rootHex = literalValue(roots[0]!.object); + if (!rootHex || !/^0x[0-9a-fA-F]{64}$/.test(rootHex)) { + return { tombstone: [], cleanupTask: [] }; + } + // Repeated finalization evidence can append several timestamp values to the + // immutable operation subject. Keep the earliest valid value for backlog + // age and normalize the replacement back to one row. + const earliestMarkedAt = markedAtRows + .map((quad) => ({ quad, value: literalValue(quad.object) })) + .filter((entry): entry is { quad: Quad; value: string } => + entry.value !== undefined && Number.isFinite(Date.parse(entry.value))) + .sort((a, b) => Date.parse(a.value) - Date.parse(b.value))[0]; + const markedAtIso = earliestMarkedAt?.value ?? new Date().toISOString(); return { - kaUal: descriptor.kaUal, - assertionVersion: descriptor.assertionVersion, - assertionGraph: descriptor.assertionGraph, - publicQuadsDigest: descriptor.publicQuadsDigest, - publicTripleCount: descriptor.publicQuadsCount, - privateMerkleRoot: descriptor.privateMerkleRoot, - privateTripleCount: descriptor.privateTripleCount, - shareOperationId: descriptor.shareOperationId, - publisherPeerId: descriptor.publisherPeerId, - accessPolicy: descriptor.accessPolicy, - allowedPeers: [...descriptor.allowedPeers], + tombstone: [roots[0]!, ...(earliestMarkedAt ? [earliestMarkedAt.quad] : [])], + cleanupTask: buildFinalizedSwmCleanupTaskQuads({ + contextGraphId: params.contextGraphId, + subGraphName: descriptor.subGraphName, + head: { + ...descriptor, + publicTripleCount: descriptor.publicQuadsCount, + }, + expectedMerkleRootHex: rootHex, + metaGraph: descriptor.metaGraph, + markedAtIso, + }), }; } -function sameBytes(left: Uint8Array | undefined, right: Uint8Array | undefined): boolean { - if (left === undefined || right === undefined) return left === right; - return left.length === right.length && left.every((byte, index) => byte === right[index]); -} - -function descriptorPrivateRoot( - descriptor: GraphScopedSwmRecoveryDescriptor, -): Uint8Array | undefined { - const value = descriptor.privateMerkleRoot?.replace(/^0x/i, ''); - if (value === undefined) return undefined; - if (!/^[0-9a-fA-F]{64}$/.test(value)) return new Uint8Array(0); - return Uint8Array.from(value.match(/.{2}/g)!.map((pair) => Number.parseInt(pair, 16))); -} - -async function isExactConfirmedVmAsset(params: { - store: TripleStore; - contextGraphId: string; - descriptor: GraphScopedSwmRecoveryDescriptor; -}): Promise { - const { store, contextGraphId, descriptor } = params; - const confirmed = await readConfirmedGraphKnowledgeAssetMetadataEnvelope(store, { - contextGraphId, - ual: descriptor.kaUal, - }); - if (confirmed.state !== 'confirmed') return false; - const { envelope } = confirmed; - const expectedPrivateRoot = descriptorPrivateRoot(descriptor); - if ( - envelope.assertionVersion !== descriptor.assertionVersion - || envelope.publicTripleCount !== descriptor.publicQuadsCount - || envelope.privateTripleCount !== descriptor.privateTripleCount - || envelope.subGraphName !== descriptor.subGraphName - || !sameBytes(envelope.privateMerkleRoot, expectedPrivateRoot) - ) { - return false; - } - - const vmResult = await store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(envelope.assertionGraph)}> { ?s ?p ?o } }`, - { - priority: 'background', - source: 'agent.sharedMemorySync.snapshotMaterializer.readConfirmedVmGraph', - }, - ); - if (vmResult.type !== 'quads') return false; - const vmQuads = vmResult.quads.map((quad) => ({ ...quad, graph: '' })); - if ( - vmQuads.length !== descriptor.publicQuadsCount - || workspacePublicQuadsDigest(vmQuads) !== descriptor.publicQuadsDigest - ) { - return false; - } - return sameBytes( - computeFlatKCRootV10( - vmQuads, - envelope.privateMerkleRoot ? [envelope.privateMerkleRoot] : [], - ), - envelope.merkleRoot, - ); -} - /** * Build the production materializer over the agent's own store, lock map and * list-cache invalidation hook. @@ -428,79 +347,17 @@ export function createSharedMemorySnapshotMaterializer(deps: { return workspacePublicQuadsDigest(stored) === descriptor.publicQuadsDigest; }, - discardFinalizedGraphAsset: async (contextGraphId, descriptor) => { - if (!await isExactConfirmedVmAsset({ + ensureFinalizedCleanupTask: async (contextGraphId, descriptor) => { + const preserved = await readExactFinalizedOperationTombstone({ store: deps.store, contextGraphId, descriptor, - })) { - return false; - } - - // The caller owns the canonical per-KA writer lock. Re-read the active - // head inside it so a newer SWM assertion can never be removed by a - // delayed finalized snapshot. - let currentHead; - try { - currentHead = await resolveKnowledgeAssetWorkspaceHead({ - store: deps.store, - graphManager: new GraphManager(deps.store), - contextGraphId, - kaUal: descriptor.kaUal, - subGraphName: descriptor.subGraphName, - queryOptions: { - priority: 'background', - source: 'agent.sharedMemorySync.snapshotMaterializer.readFinalizedSwmHead', - }, - }); - } catch { - // Exact VM still makes the incoming descriptor stale. Preserve corrupt - // local SWM state for explicit recovery, but never import another copy. - return true; - } - if ( - !currentHead - || !sameKnowledgeAssetWorkspaceHead( - currentHead, - workspaceHeadFromDescriptor(descriptor), - ) - ) { - return true; - } - - const swmResult = await deps.store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(descriptor.assertionGraph)}> { ?s ?p ?o } }`, - { - priority: 'background', - source: 'agent.sharedMemorySync.snapshotMaterializer.readFinalizedSwmGraph', - }, - ); - if (swmResult.type !== 'quads') return true; - const swmQuads = swmResult.quads.map((quad) => ({ ...quad, graph: '' })); - const swmIsAbsent = swmQuads.length === 0; - const swmIsExact = swmQuads.length === descriptor.publicQuadsCount - && workspacePublicQuadsDigest(swmQuads) === descriptor.publicQuadsDigest; - if (!swmIsAbsent && !swmIsExact) return true; - - const replaced = await tryReplaceGraphAndSubjectAtomically( - deps.store, - descriptor.assertionGraph, - [], - descriptor.metaGraph, - descriptor.headSubject, - [], - { - priority: 'background', - source: 'agent.sharedMemorySync.snapshotMaterializer.discardFinalizedSwm', - }, - ); - if (!replaced) { - throw new Error( - 'finalized SWM anti-resurrection requires atomic graph-and-head replacement support', - ); + sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + }); + if (preserved.cleanupTask.length > 0) { + await deps.insertReplacementMetadata(preserved.cleanupTask); + deps.invalidateListContextGraphsCache(); } - deps.invalidateListContextGraphsCache(); - return true; }, replaceGraph: async (graphUri, quads) => { diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index c47e59432a..0a94ce3420 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -54,6 +54,17 @@ const DKG_KA_UAL = `${DKG}kaUal`; const DKG_ASSERTION_VERSION = `${DKG}assertionVersion`; const DKG_SHARE_OPERATION_ID = `${DKG}shareOperationId`; const DKG_FINALIZED_SWM_CLEANUP_ROOT = `${DKG}finalizedSwmCleanupRoot`; +const DKG_FINALIZED_SWM_CLEANUP_MARKED_AT = `${DKG}finalizedSwmCleanupMarkedAt`; +const DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT = `${DKG}finalizedSwmCleanupHeadFingerprint`; +const DKG_FINALIZED_SWM_CLEANUP_TASK = `${DKG}FinalizedSwmCleanupTask`; +const localFinalizedSwmCleanupRowFilter = (subject: string, predicate: string): string => ` + FILTER(${predicate} NOT IN ( + <${DKG_FINALIZED_SWM_CLEANUP_ROOT}>, + <${DKG_FINALIZED_SWM_CLEANUP_MARKED_AT}>, + <${DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT}> + )) + FILTER NOT EXISTS { ${subject} <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_FINALIZED_SWM_CLEANUP_TASK}> } +`; const DKG_ASSERTION_GRAPH = `${DKG}assertionGraph`; const DKG_ASSERTION_NAME = `${DKG}assertionName`; const DKG_MEMORY_LAYER = `${DKG}memoryLayer`; @@ -919,14 +930,6 @@ export async function readSwmDataPage(params: { }): Promise { const dataGraphs = swmGraphsForRegisteredSubGraphs(params.contextGraphId, params.registeredSubGraphNames, false); const graphSet = new Set(params.graphList); - const swmMetaGraphs = dataGraphs - .map((graph) => `${graph}_meta`) - .filter((graph) => graphSet.has(graph)); - const finalizedAssertionGraphs = readFinalizedSwmAssertionGraphs( - params.store, - swmMetaGraphs, - params.signal, - ); const candidateGraphsFor = (graph: string) => params.graphList .filter((candidate) => candidate === graph || isSharedMemoryBucketDescendantDataGraph(candidate, graph)) .sort(compareCodePoint); @@ -941,10 +944,7 @@ export async function readSwmDataPage(params: { : undefined; if (!params.cutoffIso) { - const blockedGraphs = await finalizedAssertionGraphs; - const candidateGraphs = dedupeStrings(dataGraphs.flatMap(candidateGraphsFor)) - .filter((graph) => !blockedGraphs.has(graph)) - .sort(compareCodePoint); + const candidateGraphs = dedupeStrings(dataGraphs.flatMap(candidateGraphsFor)).sort(compareCodePoint); return readPagedRowsAcrossGraphs( params.store, candidateGraphs, @@ -958,13 +958,12 @@ export async function readSwmDataPage(params: { } const loadStoreBoundedPage: StorePageLoader = async (offset, limit, signal) => { - const loadPlan = async () => buildFreshSwmDataGraphPlan( + const loadPlan = () => buildFreshSwmDataGraphPlan( params.store, dataGraphs, graphSet, candidateGraphsFor, params.cutoffIso!, - await finalizedAssertionGraphs, signal, ); const plan = params.freshGraphPlanMemo && params.rowListCacheKey @@ -994,33 +993,6 @@ export async function readSwmDataPage(params: { ); } -/** - * Read the assertion graphs whose active graph-scoped SWM lifecycle has been - * finalized and durably marked for deferred cleanup. The marker itself is - * filtered from the meta phase; this companion filter keeps the corresponding - * payload graph out of both the cutoff-less exact plan and TTL data plans. - */ -async function readFinalizedSwmAssertionGraphs( - store: TripleStore, - swmMetaGraphs: readonly string[], - signal?: AbortSignal, -): Promise> { - const values = graphValues(swmMetaGraphs); - if (!values) return new Set(); - // sparql-scan-allow: R2 -- ?metaGraph is bound by the finite admitted SWM meta graph family - const result = await store.query(` - SELECT DISTINCT ?assertionGraph WHERE { - VALUES ?metaGraph { ${values} } - GRAPH ?metaGraph { - ?marked <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; - <${DKG_ASSERTION_GRAPH}> ?assertionGraph . - } - } - `, syncResponderStoreOptions(signal, 'sync.responder.readFinalizedSwmAssertionGraphs')); - if (result.type !== 'bindings') return new Set(); - return new Set(result.bindings.map((row) => row['assertionGraph']).filter(Boolean)); -} - export async function readDurableMetaPage(params: { store: TripleStore; contextGraphId: string; @@ -2695,7 +2667,7 @@ async function readBoundedSwmMetaSnapshot( return filterSwmMetaSnapshotRows(rows, null); } -export function filterSwmMetaSnapshotRows( +function filterSwmMetaSnapshotRows( rows: readonly SyncRow[], cutoffIso: string | null, ): SyncRow[] { @@ -2731,19 +2703,22 @@ export function filterSwmMetaSnapshotRows( } return keys; }; - const blockedSubjects = new Set(); - const blockedTupleKeys = new Set(); - for (const [subject] of bySubject) { - if (objects(subject, DKG_FINALIZED_SWM_CLEANUP_ROOT).length === 0) continue; - blockedSubjects.add(subject); - for (const key of tupleKeys(subject)) blockedTupleKeys.add(key); - } + // Finalized-SWM markers/tasks are local maintenance state. Strip only those + // rows; the active SWM lifecycle and its payload remain syncable until the + // independent idle GC safely removes them. + const cleanupTaskSubjects = new Set(); for (const [subject] of bySubject) { - if (tupleKeys(subject).some((key) => blockedTupleKeys.has(key))) { - blockedSubjects.add(subject); + if (objects(subject, DKG_ONTOLOGY.RDF_TYPE).includes(DKG_FINALIZED_SWM_CLEANUP_TASK)) { + cleanupTaskSubjects.add(subject); } } - const syncableRows = rows.filter((row) => !blockedSubjects.has(row.s)); + const localCleanupPredicates = new Set([ + DKG_FINALIZED_SWM_CLEANUP_ROOT, + DKG_FINALIZED_SWM_CLEANUP_MARKED_AT, + DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT, + ]); + const syncableRows = rows.filter((row) => + !cleanupTaskSubjects.has(row.s) && !localCleanupPredicates.has(row.p)); if (cutoffIso == null) return syncableRows.sort(compareRows); if (!Number.isFinite(cutoffMs)) return []; @@ -2767,8 +2742,9 @@ export function filterSwmMetaSnapshotRows( /** * Legacy TTL-unfiltered store-paged compatibility path (cutoffIso == null - * sessions only). Finalized cleanup-marked lifecycles are still excluded from - * synchronization. The former TTL variant of this query — DISTINCT + a six-predicate + * sessions only). Local finalized-cleanup metadata is filtered while the SWM + * lifecycle itself remains syncable until idle GC removes it. The former TTL + * variant of this query — DISTINCT + a six-predicate * UNION join + global `ORDER BY ?g ?s ?p ?o` re-evaluated with a growing * OFFSET per page over a mutable graph family — was the #1847 store-melter and * is deliberately DELETED, not gated: TTL-filtered sessions page from the @@ -2787,22 +2763,13 @@ async function readSwmMetaRowsPage( const swmMetaValues = graphValues(swmMetaGraphs); if (!swmMetaValues) return []; // sparql-scan-allow: R2 -- ?g is bound by a finite VALUES list of pre-admitted SWM meta graph IRIs - // sparql-scan-allow: R3 -- legacy cutoff-less compatibility lane with only finalized-lifecycle exclusion; TTL sessions page from the session plan instead (#1847) + // sparql-scan-allow: R3 -- legacy cutoff-less compatibility lane; TTL sessions page from the session plan instead (#1847) const res = await store.query(` SELECT DISTINCT ?g ?s ?p ?o WHERE { VALUES ?g { ${swmMetaValues} } GRAPH ?g { ?s ?p ?o . - FILTER NOT EXISTS { ?s <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot } - FILTER NOT EXISTS { - ?blockedHead <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; - <${DKG_KA_UAL}> ?blockedUal ; - <${DKG_ASSERTION_VERSION}> ?blockedVersion ; - <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . - ?s <${DKG_KA_UAL}> ?blockedUal ; - <${DKG_ASSERTION_VERSION}> ?blockedVersion ; - <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . - } + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} } } ORDER BY ?g ?s ?p ?o @@ -2901,15 +2868,7 @@ async function readFreshSwmMetaSubjects( SELECT DISTINCT ?s WHERE { GRAPH <${assertSafeIri(graph)}> { ?s <${DKG_PUBLISHED_AT}> ?ts . - FILTER NOT EXISTS { - ?blockedHead <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; - <${DKG_KA_UAL}> ?blockedUal ; - <${DKG_ASSERTION_VERSION}> ?blockedVersion ; - <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . - ?s <${DKG_KA_UAL}> ?blockedUal ; - <${DKG_ASSERTION_VERSION}> ?blockedVersion ; - <${DKG_SHARE_OPERATION_ID}> ?blockedShareId . - } + FILTER NOT EXISTS { ?s <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_FINALIZED_SWM_CLEANUP_TASK}> } ${cutoffFilter} } } @@ -2922,13 +2881,7 @@ async function readFreshSwmMetaSubjects( <${DKG_KA_UAL}> ?headUal ; <${DKG_ASSERTION_VERSION}> ?headVersion ; <${DKG_SHARE_OPERATION_ID}> ?shareId . - FILTER NOT EXISTS { ?s <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot } - FILTER NOT EXISTS { - ?blockedHead <${DKG_FINALIZED_SWM_CLEANUP_ROOT}> ?cleanupRoot ; - <${DKG_KA_UAL}> ?headUal ; - <${DKG_ASSERTION_VERSION}> ?headVersion ; - <${DKG_SHARE_OPERATION_ID}> ?shareId . - } + FILTER NOT EXISTS { ?s <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_FINALIZED_SWM_CLEANUP_TASK}> } ?headOperation <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_WORKSPACE_OPERATION}> ; <${DKG_CONTENT_SCOPE_VERSION}> ${GRAPH_KA_CONTENT_SCOPE_VERSION} ; <${DKG_KA_UAL}> ?headUal ; @@ -2961,7 +2914,10 @@ async function countFreshSwmMetaSubjectRows( res = await store.query(` SELECT ?s (COUNT(*) AS ?count) WHERE { VALUES ?s { ${subjectValues(chunk)} } - GRAPH <${assertSafeIri(graph)}> { ?s ?p ?o } + GRAPH <${assertSafeIri(graph)}> { + ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} + } } GROUP BY ?s `, { @@ -3108,7 +3064,10 @@ async function readFreshSwmMetaSubjectWindowRows( const res = await store.query(` SELECT ?s ?p ?o WHERE { VALUES ?s { ${subjectValues(chunk.map((entry) => entry.subject))} } - GRAPH <${assertSafeIri(graph)}> { ?s ?p ?o } + GRAPH <${assertSafeIri(graph)}> { + ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} + } } `, { ...syncResponderStoreOptions(signal, 'sync.responder.readFreshSwmMetaSubjectRows'), @@ -3383,7 +3342,6 @@ async function buildFreshSwmDataGraphPlan( graphSet: ReadonlySet, candidateGraphsFor: (graph: string) => string[], cutoffIso: string, - finalizedAssertionGraphs: ReadonlySet, signal?: AbortSignal, ): Promise { const cutoffFilter = @@ -3398,9 +3356,7 @@ async function buildFreshSwmDataGraphPlan( } const uniqueCandidates = [...new Map( candidates.map((candidate) => [candidate.graph, candidate]), - ).values()] - .filter((candidate) => !finalizedAssertionGraphs.has(candidate.graph)) - .sort((a, b) => compareCodePoint(a.graph, b.graph)); + ).values()].sort((a, b) => compareCodePoint(a.graph, b.graph)); if (uniqueCandidates.length === 0) return { entries: [], totalRows: 0 }; const rootsByGraph = new Map>(); diff --git a/packages/agent/test/agent.part-16.test.ts b/packages/agent/test/agent.part-16.test.ts index 0bc2b1e26f..e46a2e1990 100644 --- a/packages/agent/test/agent.part-16.test.ts +++ b/packages/agent/test/agent.part-16.test.ts @@ -166,11 +166,10 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => }; }); (agent as any).syncSharedMemoryFromPeerDetailed = syncSharedMemoryFromPeerDetailed; - const cleanupExpiredSharedMemory = recorder(async () => { - lifecycleOrder.push('cleanup'); - return 0; + const wakeFinalizedSwmCleanup = recorder(() => { + lifecycleOrder.push('cleanup-wake'); }); - (agent as any).cleanupExpiredSharedMemory = cleanupExpiredSharedMemory; + (agent as any).wakeFinalizedSwmCleanup = wakeFinalizedSwmCleanup; const result = await agent.syncContextGraphFromConnectedPeers('runtime-contextGraph', { includeSharedMemory: true, @@ -198,15 +197,8 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => expect(result.diagnostics.noProtocolPeers).toBe(0); expect(result.diagnostics.durable.failedPeers).toBe(0); expect(result.diagnostics.sharedMemory.failedPeers).toBe(0); - expect(cleanupExpiredSharedMemory.calls).toEqual([[ - { - finalizedOnly: true, - contextGraphIds: ['runtime-contextGraph'], - finalizedCleanupBudget: 64, - queueBehindActiveWork: true, - }, - ]]); - expect(lifecycleOrder).toEqual(['durable', 'shared-memory', 'cleanup']); + expect(wakeFinalizedSwmCleanup.calls).toEqual([[]]); + expect(lifecycleOrder).toEqual(['durable', 'shared-memory', 'cleanup-wake']); expect(agent.getSubscribedContextGraphs().get('runtime-contextGraph')).toMatchObject({ synced: true, sharedMemorySynced: true, diff --git a/packages/agent/test/finalized-swm-cleanup-worker.test.ts b/packages/agent/test/finalized-swm-cleanup-worker.test.ts new file mode 100644 index 0000000000..8be926922b --- /dev/null +++ b/packages/agent/test/finalized-swm-cleanup-worker.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from 'vitest'; +import { FinalizedSwmCleanupWorker } from '../src/finalized-swm-cleanup-worker.js'; + +describe('FinalizedSwmCleanupWorker', () => { + it('records backlog, marker age, pressure skips and deleted items', async () => { + let now = 20_000; + const sweep = vi.fn().mockResolvedValue({ + backlogDepth: 7, + oldestMarkerAt: 5_000, + deletedItems: 2, + pressureSkipped: true, + }); + const worker = new FinalizedSwmCleanupWorker({ + sweep, + now: () => now, + // Keep retry scheduling observable without running another pass. + setTimer: (() => ({ unref() {} })) as never, + clearTimer: () => {}, + }); + + await worker.runNow(); + expect(worker.snapshot()).toMatchObject({ + backlogDepth: 7, + oldestMarkerAgeMs: 15_000, + pressureSkips: 1, + deletedItems: 2, + runs: 1, + lastError: null, + }); + now += 1_000; + expect(worker.snapshot().oldestMarkerAgeMs).toBe(16_000); + await worker.close(); + }); + + it('wake is non-blocking and coalesces work while one sweep is in flight', async () => { + const timers: Array<() => void> = []; + let release!: () => void; + const pending = new Promise((resolve) => { release = resolve; }); + const sweep = vi.fn(async () => { + await pending; + return { + backlogDepth: 0, + oldestMarkerAt: null, + deletedItems: 0, + pressureSkipped: false, + }; + }); + const worker = new FinalizedSwmCleanupWorker({ + sweep, + setTimer: ((fn: () => void) => { + timers.push(fn); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + + expect(worker.wake()).toBeUndefined(); + expect(sweep).not.toHaveBeenCalled(); + timers.shift()!(); + await Promise.resolve(); + expect(sweep).toHaveBeenCalledTimes(1); + expect(worker.wake()).toBeUndefined(); + expect(worker.wake()).toBeUndefined(); + release(); + await worker.close(); + expect(sweep).toHaveBeenCalledTimes(1); + }); + + it('contains sweep errors and exposes the last failure', async () => { + const onError = vi.fn(); + const worker = new FinalizedSwmCleanupWorker({ + sweep: async () => { throw new Error('store unavailable'); }, + onError, + }); + + await expect(worker.runNow()).resolves.toBeUndefined(); + expect(onError).toHaveBeenCalledTimes(1); + expect(worker.snapshot()).toMatchObject({ + runs: 1, + lastError: 'store unavailable', + }); + await worker.close(); + }); + + it('reschedules a yielded wall-clock slice without counting it as pressure', async () => { + const timers: Array<{ fn: () => void; delayMs: number }> = []; + const worker = new FinalizedSwmCleanupWorker({ + sweep: async () => ({ + backlogDepth: 0, + oldestMarkerAt: null, + deletedItems: 0, + pressureSkipped: false, + budgetExhausted: true, + }), + retryDelayMs: 123, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + + await worker.runNow(); + expect(worker.snapshot().pressureSkips).toBe(0); + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(123); + await worker.close(); + }); +}); diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index c999555c83..ac3d072212 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -27,9 +27,11 @@ import { swmKaWriteLockKey, withKeyedLocks, workspaceOperationSubject, + workspacePublicQuadsDigest, } from '@origintrail-official/dkg-publisher'; import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, FinalizationHandler, } from '../src/finalization-handler.js'; import { @@ -43,6 +45,8 @@ import { type ChainReconcilerDeps, } from '../src/chain-reconciler.js'; import { createCursorState } from '../src/reconcile-cursor.js'; +import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped-swm-recovery.js'; +import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; const CG = 'rootless-finalization'; const AUTHOR = '0x1111111111111111111111111111111111111111'; @@ -202,6 +206,7 @@ describe('graph-scoped finalization handler', () => { message: FinalizationMessageMsg; swmGraph: string; vmGraph: string; + publicQuads: Quad[]; }> { const scope = createGraphKnowledgeAssetScope(UAL, VERSION); const swmGraph = knowledgeAssetLayerGraphUri( @@ -265,6 +270,7 @@ describe('graph-scoped finalization handler', () => { return { swmGraph, vmGraph, + publicQuads, message: { ual: scope.ual, contextGraphId: CG, @@ -429,6 +435,66 @@ describe('graph-scoped finalization handler', () => { expect(legacyRoots).toMatchObject({ type: 'boolean', value: false }); }); + it('eventually removes a late repeated snapshot after the node becomes idle again', async () => { + const { message, swmGraph, vmGraph, publicQuads } = await stageGraph(); + const metaGraph = graphManager.sharedMemoryMetaUri(CG); + const beforeFinalization = await store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${metaGraph}> { ?s ?p ?o } }`, + ); + expect(beforeFinalization.type).toBe('quads'); + if (beforeFinalization.type !== 'quads') throw new Error('expected SWM metadata'); + const [descriptor] = parseGraphScopedSwmRecoveryDescriptors({ + contextGraphId: CG, + metaQuads: beforeFinalization.quads.map((quad) => ({ ...quad, graph: metaGraph })), + }); + expect(descriptor).toBeDefined(); + if (!descriptor) throw new Error('expected graph-scoped SWM descriptor'); + + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + expect(await store.countQuads(vmGraph)).toBe(publicQuads.length); + + // A delayed/repeated snapshot lands normally. Ingest only re-arms the + // constant-size durable task; it never scans VM or deletes SWM. + const materializer = createSharedMemorySnapshotMaterializer({ + store, + writeLocks, + invalidateListContextGraphsCache: () => {}, + insertReplacementMetadata: (quads) => store.insert([...quads]), + }); + await materializer.withKaWriteLock(CG, undefined, UAL, async () => { + await materializer.ensureFinalizedCleanupTask(CG, descriptor); + await materializer.replaceGraph(swmGraph, publicQuads); + await materializer.replaceHeadMetadata(CG, descriptor); + }); + expect(await store.countQuads(swmGraph)).toBe(publicQuads.length); + expect(await store.countQuads(vmGraph)).toBe(publicQuads.length); + + // Once idle, only the dedicated GC performs the expensive verification + // and exact conditional delete. VM remains byte-for-byte intact. + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + expect(await store.countQuads(vmGraph)).toBe(publicQuads.length); + expect(await drainFinalizedSwm()).toBe(0); + await expect(store.query( + `ASK { GRAPH <${metaGraph}> { ?task ` + + ` ` + + `<${FINALIZED_SWM_CLEANUP_TASK_TYPE}> } }`, + )).resolves.toMatchObject({ type: 'boolean', value: false }); + await expect(store.query( + `ASK { GRAPH <${metaGraph}> { <${workspaceOperationSubject(CG, SHARE_ID)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + const finalVm = await store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${vmGraph}> { ?s ?p ?o } }`, + ); + expect(finalVm.type).toBe('quads'); + if (finalVm.type !== 'quads') throw new Error('expected VM content'); + expect(workspacePublicQuadsDigest(finalVm.quads.map((quad) => ({ ...quad, graph: '' })))) + .toBe(workspacePublicQuadsDigest(publicQuads.map((quad) => ({ ...quad, graph: '' })))); + }); + it('accepts adapter batch metadata when the singleton KA range matches the UAL', async () => { const { message, vmGraph } = await stageGraph(); @@ -2252,18 +2318,18 @@ describe('graph-scoped finalization handler', () => { contextGraphId: CG, kaUal: UAL, })).resolves.toBeUndefined(); - const immutableSnapshotBlockedFromSync = await store.query( + const immutableFinalizationTombstone = await store.query( `ASK { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { ` + `<${workspaceOperationSubject(CG, SHARE_ID)}> ` + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, ); - expect(immutableSnapshotBlockedFromSync).toMatchObject({ + expect(immutableFinalizationTombstone).toMatchObject({ type: 'boolean', value: true, }); }); - it('queues explicit post-catchup cleanup behind active store work', async () => { + it('never bypasses active store pressure for finalized cleanup', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2283,17 +2349,14 @@ describe('graph-scoped finalization handler', () => { }), }); + const querySpy = vi.spyOn(store, 'query'); expect(await drainFinalizedSwm()).toBe(0); - expect(await handler.cleanupFinalizedGraphScopedSwmWhenIdle({ - contextGraphId: CG, - swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), - maxCandidates: 16, - queueBehindActiveWork: true, - })).toBe(1); - expect(await store.countQuads(swmGraph)).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + expect(querySpy).not.toHaveBeenCalled(); + querySpy.mockRestore(); }); - it('discovers the WorkspaceOperation for bounded subgraph cleanup', async () => { + it('discovers the independent cleanup task for bounded subgraph cleanup', async () => { const subGraphName = 'named-cleanup'; const { message, swmGraph } = await stageGraph(undefined, subGraphName); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2313,7 +2376,7 @@ describe('graph-scoped finalization handler', () => { maxCandidates: 1, })).resolves.toBe(1); expect(discoverQueries.some((query) => query.includes( - ' ', + ' ', ))).toBe(true); expect(await store.countQuads(swmGraph)).toBe(0); await expect(resolveKnowledgeAssetWorkspaceHead({ @@ -2326,7 +2389,7 @@ describe('graph-scoped finalization handler', () => { querySpy.mockRestore(); }); - it('retries the actual post-catchup discover query after a transient scheduler timeout', async () => { + it('keeps cleanup background-only and leaves retry to the worker after scheduler pressure', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2340,12 +2403,12 @@ describe('graph-scoped finalization handler', () => { if ( !injectedBusyTimeout && options?.source === 'agent.finalization.graphScopedSwmCleanup.discover' - && query.includes('SELECT DISTINCT ?head ?ual ?version ?root ?shareId ?subGraphName') + && query.includes('SELECT DISTINCT ?task ?ual ?version ?root ?shareId') ) { injectedBusyTimeout = true; throw new StoreSchedulerBusyError( 'queue_wait_timeout', - 'normal', + 'background', options.source, ); } @@ -2356,12 +2419,11 @@ describe('graph-scoped finalization handler', () => { contextGraphId: CG, swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), maxCandidates: 16, - queueBehindActiveWork: true, - })).resolves.toBe(1); + })).rejects.toBeInstanceOf(StoreSchedulerBusyError); expect(injectedBusyTimeout).toBe(true); expect(cleanupPriorities.length).toBeGreaterThan(0); - expect(new Set(cleanupPriorities)).toEqual(new Set(['normal'])); - expect(await store.countQuads(swmGraph)).toBe(0); + expect(new Set(cleanupPriorities)).toEqual(new Set(['background'])); + expect(await store.countQuads(swmGraph)).toBe(2); querySpy.mockRestore(); }); diff --git a/packages/agent/test/swm-public-snapshot-materialization.test.ts b/packages/agent/test/swm-public-snapshot-materialization.test.ts index d9db357d15..e9e3f9bc72 100644 --- a/packages/agent/test/swm-public-snapshot-materialization.test.ts +++ b/packages/agent/test/swm-public-snapshot-materialization.test.ts @@ -117,7 +117,6 @@ function fixture(subGraphName?: string) { interface HarnessOverrides { storedHead?: () => StoredWorkspaceHeadState; contentPresent?: () => boolean; - finalized?: () => boolean; replaceImpl?: (graphUri: string, quads: Quad[]) => Promise; onLockRequested?: () => void; lockMap?: Map>; @@ -185,14 +184,13 @@ function harness(overrides: HarnessOverrides = {}) { overrides.onLockRequested?.(); return withKeyedLocks(lockMap, [swmKaWriteLockKey(contextGraphId, subGraphName, kaUal)], fn); }, + ensureFinalizedCleanupTask: async () => { + events.push('cleanup-task-ensured'); + }, isGraphAssetMaterialized: async () => { events.push('content-checked'); return overrides.contentPresent?.() ?? false; }, - discardFinalizedGraphAsset: async () => { - events.push('finalized-checked'); - return overrides.finalized?.() ?? false; - }, readStoredHead: async () => { events.push('version-read'); return overrides.storedHead?.() ?? { version: null, needsRepair: false }; @@ -313,23 +311,19 @@ describe('public SWM snapshot materialization', () => { expect(summary.failedPhases).toBe(0); }); - it('does not resurrect a snapshot that is already exactly finalized in VM', async () => { - // This is the late-sync race from the live blackbox run: cleanup drains - // SWM, then an already in-flight peer snapshot arrives. Exact VM proof - // makes that descriptor terminal, so neither data nor head metadata may - // be restored. + it('does not perform finalized cleanup while materializing a late snapshot', async () => { + // Eventual convergence deliberately permits a temporary SWM copy. The + // snapshot path performs its normal write only; the real store-backed + // metadata replacement re-arms the durable task for the independent GC. const h = harness({ - finalized: () => true, storedHead: () => ({ version: '1', needsRepair: false }), contentPresent: () => false, }); const summary = await h.run(); - expect(h.events).toContain('finalized-checked'); - expect(h.events).not.toContain('version-read'); - expect(h.events).not.toContain('content-checked'); - expect(h.events).not.toContain('replaced'); - expect(h.events).not.toContain('head-swapped'); - expect(h.inserted.every((batch) => batch.every((q) => q.graph !== WS_META))).toBe(true); + expect(h.events).toContain('version-read'); + expect(h.events).toContain('content-checked'); + expect(h.events).toContain('replaced'); + expect(h.events).toContain('head-swapped'); expect(summary.failedPhases).toBe(0); }); diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index 064fc3220e..75dc296c5d 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -31,8 +31,6 @@ import { type OperationContext, } from '@origintrail-official/dkg-core'; import { - computeFlatKCRootV10, - generateGraphKnowledgeAssetMetadata, generateKnowledgeAssetShareMetadata, resolveKnowledgeAssetWorkspaceHead, workspacePublicQuadsDigest, @@ -43,7 +41,11 @@ import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; import { runSharedMemorySync } from '../src/sync/requester/shared-memory-sync.js'; import type { SyncPageResult } from '../src/sync/requester/page-fetch.js'; -import { FINALIZED_SWM_CLEANUP_ROOT_PREDICATE } from '../src/finalization-handler.js'; +import { + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from '../src/finalization-handler.js'; const CG = 'ws00-materializer-real-store'; const WS_META = contextGraphWorkspaceMetaGraphUri(CG); @@ -114,38 +116,6 @@ function descriptorFor(fixture: typeof v1) { return descriptors[0]!; } -function confirmedVmFor(fixture: typeof v1) { - const scope = createGraphKnowledgeAssetScope(UAL, fixture.version); - const assertionGraph = knowledgeAssetLayerGraphUri(CG, MemoryLayer.VerifiableMemory, scope); - const merkleRoot = computeFlatKCRootV10(fixture.payload, []); - const agentAddress = BigInt('0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); - const batchId = (agentAddress << 96n) | 9n; - const meta = generateGraphKnowledgeAssetMetadata({ - contextGraphId: CG, - ual: UAL, - merkleRoot, - publisherPeerId: 'peer-source', - accessPolicy: 'public', - allowedPeers: [], - timestamp: new Date(0), - assertionVersion: fixture.version, - authorAddress: '0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - publicTripleCount: fixture.payload.length, - privateTripleCount: 0, - assertionGraph, - }, { - status: 'confirmed', - confirmation: { - kind: 'finalized-materialization', - provenance: { - batchId, - materializedVersion: { blockNumber: 123, txIndex: 0 }, - }, - }, - }); - return { assertionGraph, merkleRoot, meta }; -} - function materializerFor(store: TripleStore) { let invalidations = 0; const materializer = createSharedMemorySnapshotMaterializer({ @@ -169,6 +139,19 @@ async function distinctObjects(store: TripleStore, graph: string, subject: strin return result.bindings.map((row) => String(row['o'])).sort(); } +async function distinctSubjects( + store: TripleStore, + graph: string, + predicate: string, + object: string, +): Promise { + const result = await store.query( + `SELECT DISTINCT ?s WHERE { GRAPH <${graph}> { ?s <${predicate}> <${object}> } }`, + ); + if (result.type !== 'bindings') throw new Error(`unexpected ${result.type}`); + return result.bindings.map((row) => String(row['s'])).sort(); +} + describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', () => { it('shares one assertion graph across versions (the premise of the digest guard)', () => { expect(v1.assertionGraph).toBe(v2.assertionGraph); @@ -283,38 +266,69 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', expect(await distinctObjects(store, WS_META, foreignOp, `${DKG}shareOperationId`)).toEqual(['"foreign-op"']); }); - it('preserves the local finalized-cleanup token when synchronized metadata is the exact same lifecycle', async () => { + it('preserves the operation tombstone and re-arms GC for the exact lifecycle', async () => { const store = new OxigraphStore(); await store.insert([ ...v1.meta, { - subject: v1.headSubject, + subject: v1.operationSubject, predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, object: FINALIZED_ROOT, graph: WS_META, }, { subject: v1.operationSubject, - predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, - object: FINALIZED_ROOT, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: '"2026-07-31T10:00:00.000Z"^^', + graph: WS_META, + }, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: '"2026-07-31T11:00:00.000Z"^^', graph: WS_META, }, ]); const { materializer } = materializerFor(store); - await materializer.withKaWriteLock(CG, undefined, UAL, () => - materializer.replaceHeadMetadata(CG, descriptorFor(v1))); + await materializer.withKaWriteLock(CG, undefined, UAL, async () => { + const descriptor = descriptorFor(v1); + await materializer.ensureFinalizedCleanupTask(CG, descriptor); + await materializer.replaceHeadMetadata(CG, descriptor); + }); expect(await distinctObjects( store, WS_META, v1.headSubject, FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, )).toEqual([FINALIZED_ROOT]); expect(await distinctObjects( store, WS_META, v1.operationSubject, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + )).toEqual([ + '"2026-07-31T10:00:00Z"^^', + ]); + const tasks = await distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + ); + expect(tasks).toHaveLength(1); + expect(await distinctObjects(store, WS_META, tasks[0]!, `${DKG}kaUal`)).toEqual([UAL]); + expect(await distinctObjects( + store, + WS_META, + tasks[0]!, FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, )).toEqual([FINALIZED_ROOT]); }); @@ -362,71 +376,6 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', }); }); - it('atomically discards an exact SWM duplicate already confirmed in VM', async () => { - const store = new OxigraphStore(); - const vm = confirmedVmFor(v1); - await store.insert([ - ...v1.meta, - ...inGraph(v1.payload, v1.assertionGraph), - ...inGraph(v1.payload, vm.assertionGraph), - ...vm.meta, - ]); - const { materializer } = materializerFor(store); - const descriptor = descriptorFor(v1); - - const finalized = await materializer.withKaWriteLock(CG, undefined, UAL, () => - materializer.discardFinalizedGraphAsset(CG, descriptor)); - - expect(finalized).toBe(true); - expect(await materializer.isGraphAssetMaterialized(descriptor)).toBe(false); - expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) - .toEqual([]); - // Immutable operation metadata remains available for receipt/reorg - // recovery; only the active SWM graph and head are drained. - expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)) - .toEqual(['"op-v1"']); - }); - - it('rejects a finalized inbound descriptor without deleting a newer SWM lifecycle', async () => { - const store = new OxigraphStore(); - const vm = confirmedVmFor(v1); - await store.insert([ - ...v2.meta, - ...inGraph(v2.payload, v2.assertionGraph), - ...inGraph(v1.payload, vm.assertionGraph), - ...vm.meta, - ]); - const { materializer } = materializerFor(store); - - const finalized = await materializer.withKaWriteLock(CG, undefined, UAL, () => - materializer.discardFinalizedGraphAsset(CG, descriptorFor(v1))); - - expect(finalized).toBe(true); - expect(await materializer.isGraphAssetMaterialized(descriptorFor(v2))).toBe(true); - expect(await distinctObjects(store, WS_META, v2.headSubject, `${DKG}shareOperationId`)) - .toEqual(['"op-v2"']); - }); - - it('does not treat confirmed metadata as proof when the VM graph content differs', async () => { - const store = new OxigraphStore(); - const vm = confirmedVmFor(v1); - await store.insert([ - ...v1.meta, - ...inGraph(v1.payload, v1.assertionGraph), - ...inGraph(v2.payload, vm.assertionGraph), - ...vm.meta, - ]); - const { materializer } = materializerFor(store); - - const finalized = await materializer.withKaWriteLock(CG, undefined, UAL, () => - materializer.discardFinalizedGraphAsset(CG, descriptorFor(v1))); - - expect(finalized).toBe(false); - expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(true); - expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) - .toEqual(['"op-v1"']); - }); - it('replaceGraph writes atomically and invalidates the list cache', async () => { const store = new OxigraphStore(); const { materializer, invalidations } = materializerFor(store); @@ -547,33 +496,42 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', expect(h.replaceCalls()).toBe(1); }); - it('does not restore an exact finalized assertion after cleanup or a late peer sync', async () => { + it('lets a late snapshot restore temporarily and re-arms the independent GC task', async () => { const store = new OxigraphStore(); - const vm = confirmedVmFor(v1); await store.insert([ ...v1.meta, - ...inGraph(v1.payload, v1.assertionGraph), - ...inGraph(v1.payload, vm.assertionGraph), - ...vm.meta, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, ]); const h = realHarness(store, v1); const first = await h.run(); expect(first.failedPhases).toBe(0); - expect(h.replaceCalls()).toBe(0); + expect(h.replaceCalls()).toBe(1); const { materializer } = materializerFor(store); - expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(false); - expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) - .toEqual([]); + expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(true); + expect(await distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + )).toHaveLength(1); - // A second identical peer snapshot is still refused: confirmed VM is a - // durable anti-resurrection proof, not a one-shot cleanup marker. + // A repeated snapshot is a no-op because content now matches; it does not + // create duplicate tasks or add any cleanup work to the sync path. const again = await h.run(); expect(again.failedPhases).toBe(0); - expect(h.replaceCalls()).toBe(0); - expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(false); - expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) - .toEqual([]); + expect(h.replaceCalls()).toBe(1); + expect(await distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + )).toHaveLength(1); }); }); }); diff --git a/packages/agent/test/swm-ttl-v2-cleanup.test.ts b/packages/agent/test/swm-ttl-v2-cleanup.test.ts index 79dccdaf7d..5fcc25d33d 100644 --- a/packages/agent/test/swm-ttl-v2-cleanup.test.ts +++ b/packages/agent/test/swm-ttl-v2-cleanup.test.ts @@ -240,26 +240,36 @@ describe('SWM TTL cleanup of graph-scoped V2 operations', () => { it('honors an explicit public owner/name context graph during finalized cleanup', async () => { const cg = '0x1111111111111111111111111111111111111111/public-finalized-cleanup'; const cleanupFinalizedGraphScopedSwmWhenIdle = vi.fn().mockResolvedValue(0); + const inspectFinalizedGraphScopedSwmCleanupBacklog = vi.fn().mockResolvedValue({ + depth: 0, + oldestMarkerAt: null, + }); const handlerSpy = vi .spyOn(node as unknown as { getOrCreateFinalizationHandler: () => unknown }, 'getOrCreateFinalizationHandler') - .mockReturnValue({ cleanupFinalizedGraphScopedSwmWhenIdle }); + .mockReturnValue({ + cleanupFinalizedGraphScopedSwmWhenIdle, + inspectFinalizedGraphScopedSwmCleanupBacklog, + }); + const listSpy = vi.spyOn(node, 'listContextGraphs').mockResolvedValue([{ + id: cg, + uri: `did:dkg:context-graph:${cg}`, + name: 'public-finalized-cleanup', + isSystem: false, + subscribed: true, + synced: true, + }]); try { - await node.cleanupExpiredSharedMemory({ - finalizedOnly: true, - contextGraphIds: [cg], - finalizedCleanupBudget: 4, - queueBehindActiveWork: true, - }); + await node.runFinalizedSwmCleanupSweep(); } finally { handlerSpy.mockRestore(); + listSpy.mockRestore(); } - expect(cleanupFinalizedGraphScopedSwmWhenIdle).toHaveBeenCalledWith({ + expect(cleanupFinalizedGraphScopedSwmWhenIdle).toHaveBeenCalledWith(expect.objectContaining({ contextGraphId: cg, swmMetaGraph: contextGraphSharedMemoryMetaUri(cg), - maxCandidates: 4, - queueBehindActiveWork: true, - }); + maxCandidates: 1, + })); }); }); diff --git a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts index 532f0d52e6..582cb37f0f 100644 --- a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts +++ b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts @@ -657,7 +657,7 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { ...graphScopedHeadQuads(cgId, metaGraph, markedUal, 'marked', iso), { graph: metaGraph, - subject: markedHead, + subject: markedOp, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"`, }, @@ -674,8 +674,8 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { normalized.includes('ORDER BY ?g ?s ?p ?o') && /OFFSET \d+/.test(normalized) ) { - // The legacy paged query must never carry the TTL join. The only - // filters allowed here exclude finalized cleanup-marked lifecycles. + // The legacy paged query must never carry the TTL join. Its only extra + // filters strip local GC metadata, not the active SWM lifecycle. expect(normalized).not.toContain('publishedAt'); expect(normalized).toContain('finalizedSwmCleanupRoot'); legacyPagedQueries += 1; @@ -696,9 +696,10 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { 4, ); const joined = [...lines].join('\n'); - expect(lines.size).toBe(26); - expect(joined).not.toContain(markedHead); - expect(joined).not.toContain(markedOp); + expect(lines.size).toBe(37); + expect(joined).toContain(markedHead); + expect(joined).toContain(markedOp); + expect(joined).not.toContain('finalizedSwmCleanupRoot'); expect(joined).toContain(unmarkedHead); expect(joined).toContain(unmarkedOp); expect(legacyPagedQueries).toBeGreaterThan(0); diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index 5c29ab6d77..ced58f9de8 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -1,10 +1,6 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { OxigraphStore } from '@origintrail-official/dkg-storage'; import { registerSyncHandler } from '../src/sync/responder/sync-handler.js'; -import { - filterSwmMetaSnapshotRows, - type SyncRow, -} from '../src/sync/responder/graph-plan.js'; import type { SyncRequestEnvelope } from '../src/sync/auth/request-build.js'; import type { OperationContext } from '@origintrail-official/dkg-core'; @@ -67,65 +63,6 @@ const REMOTE_PEER_ID = '12D3KooWSmU3owJvB9sFw8uApDgKrv2VBMecsGGvgAc4Gq6hB57M'; const noopLog = (_ctx: OperationContext, _msg: string) => {}; -describe('finalized SWM snapshot blocking', () => { - it('blocks a marked head and its operation sibling by their shared lifecycle tuple', () => { - const graph = 'did:dkg:context-graph:tuple-block/_shared_memory_meta'; - const ual = 'did:dkg:otp:20430/0x1111111111111111111111111111111111111111/7'; - const head = `${ual}#dkg-swm-head`; - const operation = 'urn:dkg:share:tuple-block:operation-1'; - const unrelated = 'urn:dkg:share:tuple-block:unrelated'; - const tupleRows = (subject: string): SyncRow[] => [ - { g: graph, s: subject, p: `${DKG_NS}contentScopeVersion`, o: '"2"^^' }, - { g: graph, s: subject, p: `${DKG_NS}kaUal`, o: ual }, - { g: graph, s: subject, p: `${DKG_NS}assertionVersion`, o: '"1"' }, - { g: graph, s: subject, p: `${DKG_NS}shareOperationId`, o: '"operation-1"' }, - ]; - const rows: SyncRow[] = [ - ...tupleRows(head), - { g: graph, s: head, p: `${DKG_NS}finalizedSwmCleanupRoot`, o: '"sha256:abc"' }, - ...tupleRows(operation), - { g: graph, s: unrelated, p: RDF_TYPE, o: `${DKG_NS}WorkspaceOperation` }, - ]; - - const filtered = filterSwmMetaSnapshotRows(rows, null); - - expect(filtered.some((row) => row.s === head)).toBe(false); - expect(filtered.some((row) => row.s === operation)).toBe(false); - expect(filtered).toEqual([{ - g: graph, - s: unrelated, - p: RDF_TYPE, - o: `${DKG_NS}WorkspaceOperation`, - }]); - }); - - it('applies a valid freshness cutoff without a declaration-order failure', () => { - const graph = 'did:dkg:context-graph:freshness/_shared_memory_meta'; - const subject = 'urn:dkg:share:freshness:operation-1'; - const rows: SyncRow[] = [{ - g: graph, - s: subject, - p: RDF_TYPE, - o: `${DKG_NS}WorkspaceOperation`, - }, { - g: graph, - s: subject, - p: `${DKG_NS}publishedAt`, - o: '"2026-07-31T09:00:00.000Z"^^', - }]; - - const filtered = filterSwmMetaSnapshotRows( - rows, - '2026-07-31T08:00:00.000Z', - ); - expect(filtered).toHaveLength(2); - expect(new Set(filtered.map((row) => row.p))).toEqual(new Set([ - RDF_TYPE, - `${DKG_NS}publishedAt`, - ])); - }); -}); - function captureHandler(): { register: (proto: string, h: (data: Uint8Array, peerId: string) => Promise) => void; invoke: (envelope: SyncRequestEnvelope) => Promise; @@ -473,16 +410,17 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { }); it.each([0, 5_000])( - 'does not advertise finalized SWM marked for deferred cleanup (ttl=%s)', + 'keeps pending SWM syncable but never advertises local GC metadata (ttl=%s)', async (sharedMemoryTtlMs) => { const markedStore = new OxigraphStore(); const publishedAt = new Date(Date.now() - 1_000).toISOString(); const ual = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/9'; const opId = 'finalized-deferred-cleanup'; const op = `urn:dkg:share:${CG_ID}:${opId}`; + const cleanupTask = 'urn:dkg:finalized-swm-cleanup:responder-test'; const head = `${ual}#dkg-swm-head`; const assertionGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ab/9`; - const markedEntity = 'urn:swm:finalized:must-not-sync'; + const markedEntity = 'urn:swm:finalized:still-syncable'; const unmarkedGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ac/10`; const unmarkedRoot = 'urn:swm:unmarked:must-sync'; const unmarkedOp = `urn:dkg:share:${CG_ID}:unmarked-data`; @@ -495,10 +433,15 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { await markedStore.insert([ { graph: ROOT_SWM_META, subject: op, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}rootEntity`, object: markedEntity }, ...tupleRows(op), ...tupleRows(head), + ...tupleRows(cleanupTask), { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, - { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: RDF_TYPE, object: `${DKG_NS}FinalizedSwmCleanupTask` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, { graph: assertionGraph, subject: markedEntity, predicate: 'http://schema.org/name', object: '"finalized-copy"' }, { graph: unmarkedGraph, subject: unmarkedRoot, predicate: 'http://schema.org/name', object: '"live-copy"' }, { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, @@ -528,8 +471,9 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { phase: 'meta', }); - expect(out).not.toContain(head); - expect(out).not.toContain(op); + expect(out).toContain(head); + expect(out).toContain(op); + expect(out).not.toContain(cleanupTask); expect(out).not.toContain('finalizedSwmCleanupRoot'); const dataOut = await markedCap.invoke({ @@ -540,23 +484,21 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { includeSharedMemory: true, phase: 'data', }); - expect(dataOut).not.toContain(markedEntity); - expect(dataOut).not.toContain('"finalized-copy"'); + expect(dataOut).toContain(markedEntity); + expect(dataOut).toContain('"finalized-copy"'); expect(dataOut).toContain(unmarkedRoot); expect(dataOut).toContain('"live-copy"'); - // After idle cleanup removes the active head, the immutable operation - // keeps the marker for recovery but must remain outside sync. + // After idle cleanup removes the active head/task, immutable operation + // history remains syncable with only its local tombstone stripped. await markedStore.deleteByPattern({ graph: ROOT_SWM_META, subject: head, }); - await markedStore.insert([{ + await markedStore.deleteByPattern({ graph: ROOT_SWM_META, - subject: op, - predicate: `${DKG_NS}finalizedSwmCleanupRoot`, - object: `"0x${'ab'.repeat(32)}"`, - }]); + subject: cleanupTask, + }); const afterCleanup = await markedCap.invoke({ contextGraphId: CG_ID, syncSessionId: `finalized-cleanup-drained-${sharedMemoryTtlMs}`, @@ -565,7 +507,7 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { includeSharedMemory: true, phase: 'meta', }); - expect(afterCleanup).not.toContain(op); + expect(afterCleanup).toContain(op); expect(afterCleanup).not.toContain('finalizedSwmCleanupRoot'); await markedStore.close(); }, diff --git a/packages/agent/test/workspace-ttl.test.ts b/packages/agent/test/workspace-ttl.test.ts index 5c6c7b526d..517d48ff35 100644 --- a/packages/agent/test/workspace-ttl.test.ts +++ b/packages/agent/test/workspace-ttl.test.ts @@ -109,12 +109,12 @@ describe('setSharedMemoryTtlMs maintenance timer lifecycle', () => { await node.start(); await sleep(300); - // Finalized graph-scoped SWM cleanup remains active even when ordinary - // workspace TTL expiry is disabled. - expect((node as any).swmCleanupTimer).not.toBeNull(); + // Finalized graph-scoped SWM GC has its own timer; ordinary TTL expiry is + // genuinely disabled. + expect((node as any).swmCleanupTimer).toBeNull(); + expect((node as any).finalizedSwmCleanupTimer).not.toBeNull(); const store = (node as unknown as { store: TripleStore }).store; - if (!store.listGraphsByPrefix) throw new Error('test store must expose graph-prefix discovery'); let busy = true; const pressureSpy = vi.spyOn(store, 'getPressureSnapshot').mockImplementation(() => ({ ackInflight: 0, @@ -128,19 +128,23 @@ describe('setSharedMemoryTtlMs maintenance timer lifecycle', () => { maxConcurrent: 4, ackReservedSlots: 1, })); - const graphDiscoverySpy = vi.spyOn(store, 'listGraphsByPrefix'); - const discoveryCallsBeforeBusyTick = graphDiscoverySpy.mock.calls.length; + const graphDiscoverySpy = vi.spyOn(node, 'listContextGraphs'); // A TTL-disabled periodic-style call exits before graph discovery while // foreground work is active. - expect(await node.cleanupExpiredSharedMemory()).toBe(0); - expect(graphDiscoverySpy).toHaveBeenCalledTimes(discoveryCallsBeforeBusyTick); + await expect(node.runFinalizedSwmCleanupSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + expect(graphDiscoverySpy).not.toHaveBeenCalled(); // Once the store becomes idle, the same TTL-disabled maintenance path // resumes graph discovery for deferred finalized-SWM cleanup. busy = false; - await node.cleanupExpiredSharedMemory(); - expect(graphDiscoverySpy.mock.calls.length).toBeGreaterThan(discoveryCallsBeforeBusyTick); + await expect(node.runFinalizedSwmCleanupSweep()).resolves.toMatchObject({ + pressureSkipped: false, + }); + expect(graphDiscoverySpy).toHaveBeenCalledTimes(1); pressureSpy.mockRestore(); graphDiscoverySpy.mockRestore(); @@ -150,7 +154,8 @@ describe('setSharedMemoryTtlMs maintenance timer lifecycle', () => { // Disabling TTL expiry must not disable finalized-SWM maintenance. node.setSharedMemoryTtlMs(0); - expect((node as any).swmCleanupTimer).not.toBeNull(); + expect((node as any).swmCleanupTimer).toBeNull(); + expect((node as any).finalizedSwmCleanupTimer).not.toBeNull(); }, 10000); }); diff --git a/packages/agent/vitest.unit.config.ts b/packages/agent/vitest.unit.config.ts index 6edb869f43..981c6a5676 100644 --- a/packages/agent/vitest.unit.config.ts +++ b/packages/agent/vitest.unit.config.ts @@ -12,15 +12,19 @@ const SQLITE_EXEC_ARGV = [ "--no-warnings=ExperimentalWarning", ]; -// agent.part-16 pins the deterministic post-catchup finalized-SWM drain, but -// it uses the shared chain fixture. Start Hardhat when the full unit inventory -// or that file is selected. Targeted pure-unit jobs (notably the RFC-64 -// Windows gate) keep their existing fast, chain-free path. +// These lifecycle suites use the shared chain fixture. Start Hardhat when the +// full unit inventory or one of those files is selected. Targeted pure-unit +// jobs (notably the RFC-64 Windows gate) keep their existing fast, chain-free +// path. const explicitTestFilters = process.argv.filter((arg) => /(?:^|[/\\])test[/\\]/.test(arg), ); -const needsAgentPart16Fixture = explicitTestFilters.length === 0 - || explicitTestFilters.some((arg) => arg.includes("agent.part-16.test.ts")); +const needsAgentLifecycleFixture = explicitTestFilters.length === 0 + || explicitTestFilters.some((arg) => [ + "agent.part-16.test.ts", + "workspace-ttl.test.ts", + "swm-ttl-v2-cleanup.test.ts", + ].some((file) => arg.includes(file))); export default defineConfig({ test: { @@ -105,6 +109,7 @@ export default defineConfig({ "test/finalization-reconcile-negative-memo.test.ts", "test/startup-jitter.test.ts", "test/finalization-lifecycle-logger.test.ts", + "test/finalized-swm-cleanup-worker.test.ts", "test/finalization-handler.test.ts", "test/finalization-handler-chain-truth.test.ts", "test/finalization-handler-defensive-cg-id.test.ts", @@ -113,6 +118,8 @@ export default defineConfig({ "test/named-ka-publish-recovery.test.ts", "test/ka-graph-finalization-handler.test.ts", "test/agent.part-16.test.ts", + "test/workspace-ttl.test.ts", + "test/swm-ttl-v2-cleanup.test.ts", "test/sync-responder-swm-subgraphs.test.ts", "test/swm-slice-ka-bound.test.ts", "test/ka-lifecycle-asset-ual-timeout.test.ts", @@ -144,10 +151,10 @@ export default defineConfig({ "test/replace-subject-agent-wrapper.test.ts", ], testTimeout: 60_000, - globalSetup: needsAgentPart16Fixture + globalSetup: needsAgentLifecycleFixture ? ["../chain/test/hardhat-global-setup.ts"] : undefined, - env: needsAgentPart16Fixture ? { HARDHAT_PORT: "9545" } : undefined, + env: needsAgentLifecycleFixture ? { HARDHAT_PORT: "9545" } : undefined, maxWorkers: 1, pool: "forks", execArgv: SQLITE_EXEC_ARGV, diff --git a/packages/cli/src/daemon/routes/agent-chat.ts b/packages/cli/src/daemon/routes/agent-chat.ts index 44cdeb0e0f..4378356783 100644 --- a/packages/cli/src/daemon/routes/agent-chat.ts +++ b/packages/cli/src/daemon/routes/agent-chat.ts @@ -1174,6 +1174,15 @@ export function buildSloPayload(agent: { deadlineExpired: number; pending: number; }; + getFinalizedSwmCleanupStats?: () => { + backlogDepth: number; + oldestMarkerAgeMs: number | null; + pressureSkips: number; + deletedItems: number; + runs: number; + lastRunAt: string | null; + lastError: string | null; + }; }): { protocols: Record; gossip: { @@ -1220,11 +1229,21 @@ export function buildSloPayload(agent: { deadlineExpired: number; pending: number; }; + finalizedCleanup?: { + backlogDepth: number; + oldestMarkerAgeMs: number | null; + pressureSkips: number; + deletedItems: number; + runs: number; + lastRunAt: string | null; + lastError: string | null; + }; }; } { const swmHandler = agent.getSwmHandlerStats(); const substrateFanout = agent.getSwmSubstrateFanoutStats?.(); const shareAckQuorum = agent.getSwmAckQuorumStats?.(); + const finalizedCleanup = agent.getFinalizedSwmCleanupStats?.(); return { protocols: agent.getMessengerSloStats(), gossip: agent.getSwmGossipStats(), @@ -1232,6 +1251,7 @@ export function buildSloPayload(agent: { ...swmHandler, ...(substrateFanout !== undefined ? { substrateFanout } : {}), ...(shareAckQuorum !== undefined ? { shareAckQuorum } : {}), + ...(finalizedCleanup !== undefined ? { finalizedCleanup } : {}), }, }; } diff --git a/packages/cli/test/api-slo-route.test.ts b/packages/cli/test/api-slo-route.test.ts index b01cbaa3ca..3136a4473b 100644 --- a/packages/cli/test/api-slo-route.test.ts +++ b/packages/cli/test/api-slo-route.test.ts @@ -278,6 +278,45 @@ describe('/api/slo wire format (rc.9 PR-A / Codex PR #570 R10)', () => { expect((body as { swm: Record }).swm.shareAckQuorum).toBeUndefined(); }); + it('finalizedCleanup — exposes deferred GC backlog, pressure and progress', async () => { + const agent: FakeAgent = { + getMessengerSloStats: () => ({}), + getSwmGossipStats: () => ({ + publishFailures: {}, + publishFailuresOverflow: 0, + publishFailuresTruncated: false, + }), + getSwmHandlerStats: () => ({ + redundantApplies: {}, + redundantAppliesLowerBound: false, + redundantAppliesOverflow: 0, + redundantAppliesTruncated: false, + }), + getFinalizedSwmCleanupStats: () => ({ + backlogDepth: 12, + oldestMarkerAgeMs: 45_000, + pressureSkips: 7, + deletedItems: 31, + runs: 9, + lastRunAt: '2026-07-31T10:00:00.000Z', + lastError: null, + }), + }; + ({ server, port } = await startSloServer(agent)); + + const { status, body } = await get(port, '/api/slo'); + expect(status).toBe(200); + expect((body as { swm: Record }).swm.finalizedCleanup).toEqual({ + backlogDepth: 12, + oldestMarkerAgeMs: 45_000, + pressureSkips: 7, + deletedItems: 31, + runs: 9, + lastRunAt: '2026-07-31T10:00:00.000Z', + lastError: null, + }); + }); + /** * rc.9 PR-D (codex follow-up from PR-G #G1): agents that ship * the `retryable` outcome bucket (transient receiver-side From 4b1a2a9ff66c9b668088b2ac757bde43629d7eb0 Mon Sep 17 00:00:00 2001 From: Bojan Date: Fri, 31 Jul 2026 13:08:17 +0200 Subject: [PATCH 14/48] refactor(agent): isolate finalized SWM cleanup ownership --- packages/agent/src/dkg-agent-base.ts | 2 + packages/agent/src/dkg-agent-lifecycle.ts | 131 +--- packages/agent/src/dkg-agent.ts | 1 + packages/agent/src/finalization-handler.ts | 537 +-------------- .../src/finalized-swm-cleanup-service.ts | 610 ++++++++++++++++++ .../src/graph-scoped-layer-verification.ts | 89 +++ .../ka-graph-finalization-handler.test.ts | 35 +- .../agent/test/swm-ttl-v2-cleanup.test.ts | 29 +- 8 files changed, 760 insertions(+), 674 deletions(-) create mode 100644 packages/agent/src/finalized-swm-cleanup-service.ts create mode 100644 packages/agent/src/graph-scoped-layer-verification.ts diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index f667154efe..24e66be7c6 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -160,6 +160,7 @@ import { } from './auth/agent-delegation.js'; import { SyncVerifyWorker } from './sync-verify-worker.js'; import type { FinalizedSwmCleanupWorker } from './finalized-swm-cleanup-worker.js'; +import type { FinalizedSwmCleanupService } from './finalized-swm-cleanup-service.js'; import { bindRandomSampling, type RandomSamplingDisabledReason, type RandomSamplingHandle, type RandomSamplingStatus } from './random-sampling-bind.js'; import { connectToMultiaddr, ensurePeerConnected as ensurePeerConnectedAtom, primeCatchupConnections as primeCatchupConnectionsAtom } from './p2p/peer-connect.js'; import { Messenger, type SloProtocolStats } from './p2p/messenger.js'; @@ -967,6 +968,7 @@ export class DKGAgentBase { protected swmCleanupTimer: ReturnType | null = null; protected finalizedSwmCleanupTimer: ReturnType | null = null; protected finalizedSwmCleanupWorker?: FinalizedSwmCleanupWorker; + protected finalizedSwmCleanupService?: FinalizedSwmCleanupService; /** Phase B — periodic chain-driven VM reconciliation sweep timer. */ protected vmReconcileTimer: ReturnType | null = null; /** Phase B — unified per-CG coalescing and node-wide admission policy. */ diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index ea9989df44..36452411ea 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -98,7 +98,7 @@ import { pickNetworkTunables, withRetry, } from '@origintrail-official/dkg-core'; -import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type TripleStore, type TripleStoreConfig, type StorePressureSnapshot, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; import { readChangelogDeltaPage } from './sync/responder/graph-plan.js'; import { decodeChangelogRequest, encodeChangelogResponse } from './sync/changelog/wire.js'; import { runChangelogSync, planPageApply } from './sync/requester/changelog-sync.js'; @@ -343,6 +343,7 @@ import { type FinalizedSwmCleanupStats, type FinalizedSwmCleanupSweepResult, } from './finalized-swm-cleanup-worker.js'; +import { FinalizedSwmCleanupService } from './finalized-swm-cleanup-service.js'; import { reconcileContextGraph, RecentUalSet, type ChainReconcilerDeps, type OrdinalOutcome } from './chain-reconciler.js'; import { createCursorState, type CursorState } from './reconcile-cursor.js'; import { resolveStorageAckLifecycleAssetUalFromLocalSwm } from './storage-ack-lifecycle-identity.js'; @@ -379,19 +380,6 @@ type JoinApprovalRetryEntry = { lastError: string; }; -function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): boolean { - if (!snapshot) return false; - return [ - snapshot.ackInflight, - snapshot.healthInflight ?? 0, - snapshot.normalInflight, - snapshot.backgroundInflight, - snapshot.ackQueued, - snapshot.healthQueued ?? 0, - snapshot.normalQueued, - snapshot.backgroundQueued, - ].some((count) => count > 0); -} import { multiaddr } from '@multiformats/multiaddr'; import { buildCclPolicyQuads, buildPolicyApprovalQuads, buildPolicyRevocationQuads, hashCclPolicy, type CclPolicyRecord, type PolicyApprovalBinding } from './ccl-policy.js'; import { CclEvaluator, parseCclPolicy, validateCclPolicy, type CclEvaluationResult, type CclFactTuple } from './ccl-evaluator.js'; @@ -7638,109 +7626,28 @@ export class LifecycleSyncMethods extends DKGAgentBase { return this.getOrCreateFinalizedSwmCleanupWorker().snapshot(); } - /** - * One small idle-only GC slice. Pressure is checked before every discovery - * boundary and between candidates; foreground ACK, health and normal work - * always wins. - */ - async runFinalizedSwmCleanupSweep(this: DKGAgent): Promise { - const prior = this.finalizedSwmCleanupWorker?.snapshot(); - const priorOldest = prior?.oldestMarkerAgeMs == null - ? null - : Date.now() - prior.oldestMarkerAgeMs; - const pressureResult = (): FinalizedSwmCleanupSweepResult => ({ - backlogDepth: prior?.backlogDepth ?? 0, - oldestMarkerAt: priorOldest, - deletedItems: 0, - pressureSkipped: true, - }); - const budgetResult = ( - backlogDepth: number, - oldestMarkerAt: number | null, - deletedItems: number, - ): FinalizedSwmCleanupSweepResult => ({ - backlogDepth: Math.max(backlogDepth, prior?.backlogDepth ?? 0), - oldestMarkerAt: oldestMarkerAt ?? priorOldest, - deletedItems, - pressureSkipped: false, - budgetExhausted: true, - }); - const underPressure = () => hasActiveStorePressure(this.store.getPressureSnapshot?.()); - if (underPressure()) return pressureResult(); - - const deadline = Date.now() + 10_000; - const deadlineSignal = AbortSignal.timeout(10_000); - let remaining = 4; - let deletedItems = 0; - let backlogDepth = 0; - let oldestMarkerAt: number | null = null; - // Do not even enumerate CGs while the store is busy. - if (underPressure()) return pressureResult(); - const contextGraphs = (await this.listContextGraphs()).map((row) => row.id); - for (const contextGraphId of contextGraphs) { - if (underPressure()) return { ...pressureResult(), deletedItems }; - if (Date.now() >= deadline) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } - // Per-CG SWM meta discovery is also forbidden under pressure. - const metaGraphs = await listSharedMemoryMetaGraphs(this.store, contextGraphId); - for (const swmMetaGraph of metaGraphs) { - if (underPressure()) return { ...pressureResult(), deletedItems }; - if (Date.now() >= deadline) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } - if (remaining > 0) { - let cleaned: number; - try { - cleaned = await this.getOrCreateFinalizationHandler() - .cleanupFinalizedGraphScopedSwmWhenIdle({ - contextGraphId, - swmMetaGraph, - maxCandidates: 1, - signal: deadlineSignal, - }); - } catch (error) { - if (deadlineSignal.aborted) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } - throw error; - } - deletedItems += cleaned; - remaining -= 1; - } - if (underPressure()) return { ...pressureResult(), deletedItems }; - if (Date.now() >= deadline) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } - let backlog: { depth: number; oldestMarkerAt: number | null }; - try { - backlog = await this.getOrCreateFinalizationHandler() - .inspectFinalizedGraphScopedSwmCleanupBacklog({ - swmMetaGraph, - signal: deadlineSignal, - }); - } catch (error) { - if (deadlineSignal.aborted) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } - throw error; - } - backlogDepth += backlog.depth; - if ( - backlog.oldestMarkerAt !== null - && (oldestMarkerAt === null || backlog.oldestMarkerAt < oldestMarkerAt) - ) { - oldestMarkerAt = backlog.oldestMarkerAt; - } - await new Promise((resolve) => setImmediate(resolve)); - } + getOrCreateFinalizedSwmCleanupService(this: DKGAgent): FinalizedSwmCleanupService { + if (!this.finalizedSwmCleanupService) { + this.finalizedSwmCleanupService = new FinalizedSwmCleanupService({ + store: this.store, + writeLocks: this.writeLocks, + eventBus: this.eventBus, + listContextGraphIds: async () => (await this.listContextGraphs()).map((row) => row.id), + listSharedMemoryMetaGraphs: (contextGraphId) => + listSharedMemoryMetaGraphs(this.store, contextGraphId), + }); } - return { backlogDepth, oldestMarkerAt, deletedItems, pressureSkipped: false }; + return this.finalizedSwmCleanupService; + } + + /** Run one small idle-only GC slice in the dedicated cleanup service. */ + async runFinalizedSwmCleanupSweep(this: DKGAgent): Promise { + return this.getOrCreateFinalizedSwmCleanupService().runSweep(); } /** * Remove expired shared-memory operations. Finalized-SWM lifecycle GC is - * owned exclusively by FinalizedSwmCleanupWorker above. + * owned exclusively by FinalizedSwmCleanupService and its scheduler above. */ async cleanupExpiredSharedMemory(this: DKGAgent): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 139ecac78a..5454d0bb6b 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -1640,6 +1640,7 @@ export class DKGAgent extends DKGAgentBase { await this.finalizedSwmCleanupWorker.close(); this.finalizedSwmCleanupWorker = undefined; } + this.finalizedSwmCleanupService = undefined; if (this.hostModeReconcilerTimer) { clearInterval(this.hostModeReconcilerTimer); this.hostModeReconcilerTimer = null; diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index b5250c7653..7b3365228e 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -28,7 +28,6 @@ import { type GraphWriteGenSource, type QueryOptions, type SharedMemoryResultBudget, - type StorePressureSnapshot, type SwmKaGraphBound, type TripleStore, type Quad, @@ -47,13 +46,9 @@ import { compareMaterializedVersion, readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, materializedVersionQuad, withMaterializationLock, - withKeyedLocks, - swmKaWriteLockKey, KnowledgeAssetWorkspaceHeadCorruptError, resolveKnowledgeAssetOperationPublicQuads, resolveKnowledgeAssetWorkspaceHead, - sameKnowledgeAssetWorkspaceHead, - workspaceKnowledgeAssetHeadSubject, workspaceOperationSubject, workspacePublicQuadsDigest, type MaterializedVersion, @@ -63,19 +58,6 @@ import { } from '@origintrail-official/dkg-publisher'; const DKG_NS = 'http://dkg.io/ontology/'; const PROV_NS = 'http://www.w3.org/ns/prov#'; -function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): boolean { - if (!snapshot) return false; - return [ - snapshot.ackInflight, - snapshot.healthInflight ?? 0, - snapshot.normalInflight, - snapshot.backgroundInflight, - snapshot.ackQueued, - snapshot.healthQueued ?? 0, - snapshot.normalQueued, - snapshot.backgroundQueued, - ].some((count) => count > 0); -} // Slow-query / canary tags for the finalization SWM slice (#1549). A healthy fleet // sees `.fallbackUnbounded` at ~0 relative to `.bounded`; a spike means the bound is @@ -116,10 +98,8 @@ import { } from './finalization-graph-envelope.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; import { - FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, - FINALIZED_SWM_CLEANUP_TASK_TYPE, } from './dkg-agent-constants.js'; export { FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, @@ -129,8 +109,11 @@ export { } from './dkg-agent-constants.js'; import { buildFinalizedSwmCleanupTaskQuads, - finalizedSwmCleanupHeadFingerprint, } from './finalized-swm-cleanup-marker.js'; +import { + verifyExactGraphScopedLayer, + type ExactGraphScopedLayerVerification, +} from './graph-scoped-layer-verification.js'; /** * Predicate for the durable per-root keep-root-copy signal the publisher @@ -215,27 +198,6 @@ function equalBytes(left: Uint8Array, right: Uint8Array): boolean { && left.every((byte, index) => byte === right[index]); } -type ExactGraphScopedLayerVerification = - | { - status: 'verified'; - graphUri: string; - quads: Quad[]; - merkleRoot: Uint8Array; - } - | { - status: 'count-mismatch'; - graphUri: string; - actualCount: number; - } - | { - status: 'merkle-mismatch'; - graphUri: string; - } - | { - status: 'head-mismatch'; - graphUri: string; - }; - type GraphScopedMaterializationEnvelope = Pick< KnowledgeAssetWorkspaceHead, | 'publicTripleCount' @@ -1386,36 +1348,7 @@ export class FinalizationHandler { subGraphName?: string; queryOptions?: QueryOptions; }): Promise { - const graphUri = knowledgeAssetLayerGraphUri( - input.contextGraphId, - input.layer, - input.scope, - input.subGraphName, - ); - const result = await this.store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(graphUri)}> { ?s ?p ?o } }`, - input.queryOptions, - ); - const quads = result.type === 'quads' - ? result.quads.map((quad) => ({ ...quad, graph: '' })) - : []; - if (quads.length !== input.publicTripleCount) { - return { status: 'count-mismatch', graphUri, actualCount: quads.length }; - } - const merkleRoot = computeFlatKCRoot( - quads, - input.privateMerkleRoot ? [input.privateMerkleRoot] : [], - ); - if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { - return { status: 'merkle-mismatch', graphUri }; - } - if ( - input.expectedPublicQuadsDigest !== undefined - && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest - ) { - return { status: 'head-mismatch', graphUri }; - } - return { status: 'verified', graphUri, quads, merkleRoot }; + return verifyExactGraphScopedLayer({ store: this.store, ...input }); } /** @@ -1628,466 +1561,6 @@ export class FinalizationHandler { return 'marked'; } - /** Read operator backlog gauges without loading any payload graph. */ - async inspectFinalizedGraphScopedSwmCleanupBacklog(input: { - swmMetaGraph: string; - signal?: AbortSignal; - }): Promise<{ depth: number; oldestMarkerAt: number | null }> { - if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) { - throw new StoreSchedulerBusyError( - 'queue_full', - 'background', - 'agent.finalization.graphScopedSwmCleanup.backlog', - ); - } - const result = await this.store.query( - `SELECT (COUNT(DISTINCT ?task) AS ?count) (MIN(?markedAt) AS ?oldest) WHERE { - GRAPH <${assertSafeIri(input.swmMetaGraph)}> { - ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; - <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . - OPTIONAL { ?task <${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } - } - }`, - { - priority: 'background', - source: 'agent.finalization.graphScopedSwmCleanup.backlog', - signal: input.signal, - }, - ); - if (result.type !== 'bindings' || result.bindings.length === 0) { - return { depth: 0, oldestMarkerAt: null }; - } - const depth = Number.parseInt(stripOptionalLiteral(result.bindings[0]?.['count']) ?? '0', 10); - const oldestValue = stripOptionalLiteral(result.bindings[0]?.['oldest']); - const oldestMarkerAt = oldestValue ? Date.parse(oldestValue) : Number.NaN; - return { - depth: Number.isSafeInteger(depth) && depth > 0 ? depth : 0, - oldestMarkerAt: Number.isFinite(oldestMarkerAt) ? oldestMarkerAt : null, - }; - } - - /** - * Drain a bounded number of durable finalized-SWM markers only while the - * store scheduler reports no queued or in-flight work. - * - * The marker survives restart. A newer head causes the worker to retire the - * obsolete task without touching that newer lifecycle. All maintenance - * queries run in the background lane and the destructive step re-enters the - * canonical per-KA writer lock only for the final head/task re-read and - * conditional delete. - */ - async cleanupFinalizedGraphScopedSwmWhenIdle(input: { - contextGraphId: string; - swmMetaGraph: string; - maxCandidates?: number; - signal?: AbortSignal; - }): Promise { - if (!this.writeLocks) return 0; - const pressure = this.store.getPressureSnapshot?.(); - if (hasActiveStorePressure(pressure)) { - return 0; - } - const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); - const cleanupQueryOptions: QueryOptions = { - priority: 'background', - source: 'agent.finalization.graphScopedSwmCleanup.discover', - signal: input.signal, - }; - const result = await this.store.query( - `SELECT DISTINCT ?task ?ual ?version ?root ?shareId ?assertionGraph ?headFingerprint ?subGraphName WHERE { - GRAPH <${assertSafeIri(input.swmMetaGraph)}> { - ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; - <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; - <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ?headFingerprint ; - <${DKG_NS}kaUal> ?ual ; - <${DKG_NS}assertionVersion> ?version ; - <${DKG_NS}shareOperationId> ?shareId ; - <${DKG_NS}assertionGraph> ?assertionGraph . - OPTIONAL { ?task <${DKG_NS}subGraphName> ?subGraphName } - } - } ORDER BY ?task LIMIT ${limit}`, - cleanupQueryOptions, - ); - if (result.type !== 'bindings') return 0; - - let cleared = 0; - for (const row of result.bindings) { - const currentPressure = this.store.getPressureSnapshot?.(); - if (hasActiveStorePressure(currentPressure)) { - break; - } - const ual = row['ual']; - const taskSubject = row['task']; - const rawVersion = stripOptionalLiteral(row['version']); - const rawRoot = stripOptionalLiteral(row['root']); - const assertionGraph = row['assertionGraph']; - const expectedHeadFingerprint = stripOptionalLiteral(row['headFingerprint']); - const shareOperationId = stripOptionalLiteral(row['shareId']); - const subGraphName = stripOptionalLiteral(row['subGraphName']); - if ( - !taskSubject - || !taskSubject.startsWith('urn:dkg:finalized-swm-cleanup:') - || !ual - || !rawVersion - || !/^\d+$/.test(rawVersion) - || !rawRoot - || !assertionGraph - || !expectedHeadFingerprint - || !shareOperationId - ) continue; - let scope: ReturnType; - let expectedMerkleRoot: Uint8Array; - try { - scope = createGraphKnowledgeAssetScope(ual, BigInt(rawVersion)); - expectedMerkleRoot = ethers.getBytes(rawRoot); - } catch { - continue; - } - if ( - scope.ual !== ual - || expectedMerkleRoot.length !== 32 - ) { - continue; - } - const graphManager = new GraphManager(this.store); - let expectedHead: KnowledgeAssetWorkspaceHead | undefined; - try { - expectedHead = await resolveKnowledgeAssetWorkspaceHead({ - store: this.store, - graphManager, - contextGraphId: input.contextGraphId, - kaUal: scope.ual, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - } catch (error) { - if (error instanceof StoreSchedulerBusyError) { - break; - } - continue; - } - if (!expectedHead) { - await this.retireStaleFinalizedSwmCleanupTask({ - contextGraphId: input.contextGraphId, - swmMetaGraph: input.swmMetaGraph, - taskSubject, - scope, - assertionGraph, - shareOperationId, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - continue; - } - if ( - expectedHead.assertionVersion !== scope.assertionVersion - || expectedHead.shareOperationId !== shareOperationId - || expectedHead.assertionGraph !== assertionGraph - ) { - await this.retireStaleFinalizedSwmCleanupTask({ - contextGraphId: input.contextGraphId, - swmMetaGraph: input.swmMetaGraph, - taskSubject, - scope, - assertionGraph, - shareOperationId, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - continue; - } - let privateMerkleRoot: Uint8Array | undefined; - try { - privateMerkleRoot = expectedHead.privateMerkleRoot - ? ethers.getBytes(expectedHead.privateMerkleRoot) - : undefined; - } catch { - continue; - } - const outcome = await this.clearMarkedFinalizedGraphScopedSwm({ - contextGraphId: input.contextGraphId, - scope, - taskSubject, - expectedHeadFingerprint, - expectedHead, - expectedMerkleRoot, - privateMerkleRoot, - subGraphName, - queryPriority: cleanupQueryOptions.priority, - ctx: createOperationContext('system'), - }); - if (outcome === 'cleared') cleared += 1; - } - return cleared; - } - - /** - * Retire an obsolete task under the same per-KA writer lock used by SWM - * materialization. This is the short lock-held re-read/conditional-delete - * step: no VM graph read, digest, or Merkle work occurs here. - */ - private async retireStaleFinalizedSwmCleanupTask(input: { - contextGraphId: string; - swmMetaGraph: string; - taskSubject: string; - scope: ReturnType; - assertionGraph: string; - shareOperationId: string; - subGraphName?: string; - queryOptions: QueryOptions; - }): Promise { - if (!this.writeLocks) return; - const lockKey = swmKaWriteLockKey( - input.contextGraphId, - input.subGraphName, - input.scope.ual, - ); - await withKeyedLocks(this.writeLocks, [lockKey], async () => { - let currentHead: KnowledgeAssetWorkspaceHead | undefined; - try { - currentHead = await resolveKnowledgeAssetWorkspaceHead({ - store: this.store, - graphManager: new GraphManager(this.store), - contextGraphId: input.contextGraphId, - kaUal: input.scope.ual, - subGraphName: input.subGraphName, - queryOptions: input.queryOptions, - }); - } catch (error) { - if (error instanceof KnowledgeAssetWorkspaceHeadCorruptError) return; - throw error; - } - if ( - currentHead - && currentHead.assertionVersion === input.scope.assertionVersion - && currentHead.shareOperationId === input.shareOperationId - && currentHead.assertionGraph === input.assertionGraph - ) { - return; - } - if (!currentHead) { - // An interrupted snapshot metadata replacement can leave payload plus - // task without a head. Preserve the task until the retry restores its - // metadata; retire only when both mutable pieces are absent. - const payloadPresent = await this.store.query( - `ASK { GRAPH <${assertSafeIri(input.assertionGraph)}> { ?s ?p ?o } }`, - { - ...input.queryOptions, - source: 'agent.finalization.graphScopedSwmCleanup.checkHeadlessPayload', - }, - ); - if (payloadPresent.type !== 'boolean' || payloadPresent.value) return; - } - await this.store.deleteByPattern( - { graph: input.swmMetaGraph, subject: input.taskSubject }, - { - ...input.queryOptions, - source: currentHead - ? 'agent.finalization.graphScopedSwmCleanup.retireSupersededTask' - : 'agent.finalization.graphScopedSwmCleanup.retireAbsentTask', - }, - ); - }); - } - - /** Atomically remove only the still-marked, still-exact active SWM lifecycle. */ - private async clearMarkedFinalizedGraphScopedSwm(input: { - contextGraphId: string; - scope: ReturnType; - taskSubject: string; - expectedHeadFingerprint: string; - expectedHead: KnowledgeAssetWorkspaceHead; - expectedMerkleRoot: Uint8Array; - privateMerkleRoot?: Uint8Array; - subGraphName?: string; - queryPriority?: QueryOptions['priority']; - ctx: OperationContext; - }): Promise<'cleared' | 'absent' | 'preserved'> { - if (!this.writeLocks) return 'preserved'; - const { - contextGraphId, - scope, - taskSubject, - expectedHeadFingerprint, - expectedHead, - expectedMerkleRoot, - privateMerkleRoot, - subGraphName, - queryPriority, - ctx, - } = input; - const graphManager = new GraphManager(this.store); - const metaGraph = graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); - const headSubject = workspaceKnowledgeAssetHeadSubject(scope.ual); - const cleanupRootObject = JSON.stringify( - ethers.hexlify(expectedMerkleRoot).toLowerCase(), - ); - const cleanupQueryOptions: QueryOptions = { - priority: queryPriority ?? 'background', - source: 'agent.finalization.graphScopedSwmCleanup', - }; - if (finalizedSwmCleanupHeadFingerprint(expectedHead) !== expectedHeadFingerprint) { - this.log.warn(ctx, `Finalization cleanup: preserving ${scope.ual}; cleanup task fingerprint differs`); - return 'preserved'; - } - - // Expensive graph reads and hashing happen before the writer lock. The - // write-generation snapshot proves that no local writer changed any graph - // in this CG between verification and the final lock-held commit. - const writePrefix = `${contextGraphDataUri(contextGraphId)}/`; - const preflightWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); - if (queryPriority !== 'normal' && hasActiveStorePressure(this.store.getPressureSnapshot?.())) { - return 'preserved'; - } - const vmVerification = await this.verifyExactGraphScopedLayer({ - contextGraphId, - scope, - layer: MemoryLayer.VerifiableMemory, - publicTripleCount: expectedHead.publicTripleCount, - privateMerkleRoot, - expectedMerkleRoot, - expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - if (vmVerification.status !== 'verified') { - this.log.warn( - ctx, - `Finalization cleanup: preserving graph-scoped SWM for ${scope.ual}; ` - + `VM no longer matches the cleanup token (${vmVerification.status})`, - ); - return 'preserved'; - } - if (queryPriority !== 'normal' && hasActiveStorePressure(this.store.getPressureSnapshot?.())) { - return 'preserved'; - } - const swmVerification = await this.verifyExactGraphScopedLayer({ - contextGraphId, - scope, - layer: MemoryLayer.SharedWorkingMemory, - publicTripleCount: expectedHead.publicTripleCount, - privateMerkleRoot, - expectedMerkleRoot, - expectedPublicQuadsDigest: expectedHead.publicQuadsDigest, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - if ( - swmVerification.status !== 'verified' - && !(swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) - ) { - this.log.warn( - ctx, - `Finalization cleanup: preserving graph-scoped SWM for ${scope.ual}; ` - + `the current assertion no longer matches the finalized source (${swmVerification.status})`, - ); - return 'preserved'; - } - const verifiedWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); - if ( - preflightWriteGen !== undefined - && verifiedWriteGen !== preflightWriteGen - ) { - return 'preserved'; - } - - const lockKey = swmKaWriteLockKey(contextGraphId, subGraphName, scope.ual); - const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { - if ( - verifiedWriteGen !== undefined - && this.graphWriteGen?.getWriteGen(writePrefix) !== verifiedWriteGen - ) { - return 'preserved' as const; - } - let currentHead: KnowledgeAssetWorkspaceHead | undefined; - try { - currentHead = await resolveKnowledgeAssetWorkspaceHead({ - store: this.store, - graphManager, - contextGraphId, - kaUal: scope.ual, - subGraphName, - queryOptions: cleanupQueryOptions, - }); - } catch (error) { - if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; - this.log.warn( - ctx, - `Finalization: preserving graph-scoped SWM for ${scope.ual}; ` - + `the current workspace head is corrupt: ${error.message}`, - ); - return 'preserved' as const; - } - if (!currentHead) { - if (swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) { - await this.store.deleteByPattern( - { graph: metaGraph, subject: taskSubject }, - { ...cleanupQueryOptions, source: 'agent.finalization.graphScopedSwmCleanup.retireAbsentTask' }, - ); - return 'absent' as const; - } - return 'preserved' as const; - } - if ( - !sameKnowledgeAssetWorkspaceHead(currentHead, expectedHead) - || finalizedSwmCleanupHeadFingerprint(currentHead) !== expectedHeadFingerprint - ) { - this.log.info( - ctx, - `Finalization: preserving newer graph-scoped SWM lifecycle for ${scope.ual}`, - ); - return 'preserved' as const; - } - const marker = await this.store.query( - `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` - + `<${assertSafeIri(taskSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` - + `${cleanupRootObject} ; <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ` - + `${JSON.stringify(expectedHeadFingerprint)} } }`, - cleanupQueryOptions, - ); - if (marker.type !== 'boolean' || !marker.value) return 'preserved' as const; - - const replaced = await tryReplaceGraphAndSubjectAtomically( - this.store, - swmVerification.graphUri, - [], - metaGraph, - headSubject, - [], - cleanupQueryOptions, - ); - if (!replaced) { - throw Object.assign( - new Error('Graph-scoped SWM finalization cleanup requires atomic graph-and-head replacement support'), - { code: 'SWM_ATOMIC_CLEANUP_UNSUPPORTED' }, - ); - } - // Delete the independent task last. A crash after the data/head commit - // leaves only a harmless retry; deleting the task first could lose work. - await this.store.deleteByPattern( - { graph: metaGraph, subject: taskSubject }, - { ...cleanupQueryOptions, source: 'agent.finalization.graphScopedSwmCleanup.retireTask' }, - ); - return swmVerification.status === 'verified' ? 'cleared' as const : 'absent' as const; - }); - - if (outcome === 'cleared') { - this.eventBus?.emit(DKGEvent.MEMORY_GRAPH_CHANGED, { - contextGraphId, - layers: ['swm'], - subGraphName, - operation: 'shared_working_memory_finalized', - source: 'background-cleanup', - counts: { triples: expectedHead.publicTripleCount }, - }); - this.log.info( - ctx, - `Finalization cleanup: cleared finalized graph-scoped SWM assertion ${scope.ual}`, - ); - } - return outcome; - } - /** Recognize exact confirmed VM state from surviving immutable metadata. */ private async reconcileConfirmedGraphScopedVmWithoutWorkspaceHead(input: { contextGraphId: string; diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts new file mode 100644 index 0000000000..f45c0ba8ed --- /dev/null +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { + DKGEvent, + Logger, + MemoryLayer, + assertSafeIri, + contextGraphDataUri, + createGraphKnowledgeAssetScope, + createOperationContext, + type EventBus, +} from '@origintrail-official/dkg-core'; +import { + GraphManager, + StoreSchedulerBusyError, + asGraphWriteGenSource, + tryReplaceGraphAndSubjectAtomically, + type GraphWriteGenSource, + type QueryOptions, + type StorePressureSnapshot, + type TripleStore, +} from '@origintrail-official/dkg-storage'; +import { + KnowledgeAssetWorkspaceHeadCorruptError, + resolveKnowledgeAssetWorkspaceHead, + sameKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, + workspaceKnowledgeAssetHeadSubject, + type KnowledgeAssetWorkspaceHead, +} from '@origintrail-official/dkg-publisher'; +import { ethers } from 'ethers'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; +import { finalizedSwmCleanupHeadFingerprint } from './finalized-swm-cleanup-marker.js'; +import type { FinalizedSwmCleanupSweepResult } from './finalized-swm-cleanup-worker.js'; +import { verifyExactGraphScopedLayer } from './graph-scoped-layer-verification.js'; + +function stripOptionalLiteral(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const typedLiteral = value.match(/^"([\s\S]*)"\^\^<[^>]+>$/); + if (typedLiteral) return typedLiteral[1]; + const plainLiteral = value.match(/^"([\s\S]*)"$/); + return plainLiteral?.[1] ?? value; +} + +function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): boolean { + if (!snapshot) return false; + return [ + snapshot.ackInflight, + snapshot.healthInflight ?? 0, + snapshot.normalInflight, + snapshot.backgroundInflight, + snapshot.ackQueued, + snapshot.healthQueued ?? 0, + snapshot.normalQueued, + snapshot.backgroundQueued, + ].some((count) => count > 0); +} + +export interface FinalizedSwmCleanupServiceOptions { + store: TripleStore; + writeLocks?: Map>; + eventBus?: EventBus; + listContextGraphIds: () => Promise; + listSharedMemoryMetaGraphs: (contextGraphId: string) => Promise; + now?: () => number; + maxCandidatesPerSweep?: number; + wallClockBudgetMs?: number; +} + +/** + * The sole owner of finalized-SWM discovery, payload verification and deletion. + * Finalization and synchronization may create/re-arm durable tasks, but neither + * path calls this service or waits for its store work. + */ +export class FinalizedSwmCleanupService { + private readonly store: TripleStore; + private readonly writeLocks: Map> | undefined; + private readonly eventBus: EventBus | undefined; + private readonly listContextGraphIds: () => Promise; + private readonly listSharedMemoryMetaGraphs: (contextGraphId: string) => Promise; + private readonly now: () => number; + private readonly maxCandidatesPerSweep: number; + private readonly wallClockBudgetMs: number; + private readonly graphWriteGen: GraphWriteGenSource | null; + private readonly log = new Logger('FinalizedSwmCleanupService'); + private lastKnownBacklogDepth = 0; + private lastKnownOldestMarkerAt: number | null = null; + + constructor(options: FinalizedSwmCleanupServiceOptions) { + this.store = options.store; + this.writeLocks = options.writeLocks; + this.eventBus = options.eventBus; + this.listContextGraphIds = options.listContextGraphIds; + this.listSharedMemoryMetaGraphs = options.listSharedMemoryMetaGraphs; + this.now = options.now ?? Date.now; + this.maxCandidatesPerSweep = Math.min( + 16, + Math.max(1, Math.floor(options.maxCandidatesPerSweep ?? 4)), + ); + this.wallClockBudgetMs = Math.max(1, Math.floor(options.wallClockBudgetMs ?? 10_000)); + this.graphWriteGen = asGraphWriteGenSource(options.store); + } + + /** Run one bounded, idle-only maintenance slice. */ + async runSweep(): Promise { + const pressureResult = (deletedItems = 0): FinalizedSwmCleanupSweepResult => ({ + backlogDepth: this.lastKnownBacklogDepth, + oldestMarkerAt: this.lastKnownOldestMarkerAt, + deletedItems, + pressureSkipped: true, + }); + const budgetResult = ( + backlogDepth: number, + oldestMarkerAt: number | null, + deletedItems: number, + ): FinalizedSwmCleanupSweepResult => ({ + backlogDepth: Math.max(backlogDepth, this.lastKnownBacklogDepth), + oldestMarkerAt: oldestMarkerAt ?? this.lastKnownOldestMarkerAt, + deletedItems, + pressureSkipped: false, + budgetExhausted: true, + }); + const underPressure = () => hasActiveStorePressure(this.store.getPressureSnapshot?.()); + if (underPressure()) return pressureResult(); + + const deadline = this.now() + this.wallClockBudgetMs; + const deadlineSignal = AbortSignal.timeout(this.wallClockBudgetMs); + let remaining = this.maxCandidatesPerSweep; + let deletedItems = 0; + let backlogDepth = 0; + let oldestMarkerAt: number | null = null; + + // Pressure is checked before every discovery boundary, including the first + // potentially expensive context-graph enumeration. + if (underPressure()) return pressureResult(); + const contextGraphIds = await this.listContextGraphIds(); + for (const contextGraphId of contextGraphIds) { + if (underPressure()) return pressureResult(deletedItems); + if (this.now() >= deadline) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + const metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId); + for (const swmMetaGraph of metaGraphs) { + if (underPressure()) return pressureResult(deletedItems); + if (this.now() >= deadline) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + if (remaining > 0) { + let cleanup: { deletedItems: number; examinedCandidates: number }; + try { + cleanup = await this.cleanupMetaGraph({ + contextGraphId, + swmMetaGraph, + maxCandidates: remaining, + signal: deadlineSignal, + }); + } catch (error) { + if (deadlineSignal.aborted) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + throw error; + } + deletedItems += cleanup.deletedItems; + remaining -= cleanup.examinedCandidates; + } + if (underPressure()) return pressureResult(deletedItems); + if (this.now() >= deadline) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + let backlog: { depth: number; oldestMarkerAt: number | null }; + try { + backlog = await this.inspectBacklog(swmMetaGraph, deadlineSignal); + } catch (error) { + if (deadlineSignal.aborted) { + return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + } + throw error; + } + backlogDepth += backlog.depth; + if ( + backlog.oldestMarkerAt !== null + && (oldestMarkerAt === null || backlog.oldestMarkerAt < oldestMarkerAt) + ) { + oldestMarkerAt = backlog.oldestMarkerAt; + } + await new Promise((resolve) => setImmediate(resolve)); + } + } + + this.lastKnownBacklogDepth = backlogDepth; + this.lastKnownOldestMarkerAt = oldestMarkerAt; + return { backlogDepth, oldestMarkerAt, deletedItems, pressureSkipped: false }; + } + + /** Narrow test seam for one already-known SWM metadata graph. */ + async cleanupKnownMetaGraph(input: { + contextGraphId: string; + swmMetaGraph: string; + maxCandidates?: number; + signal?: AbortSignal; + }): Promise { + return (await this.cleanupMetaGraph(input)).deletedItems; + } + + private async inspectBacklog( + swmMetaGraph: string, + signal?: AbortSignal, + ): Promise<{ depth: number; oldestMarkerAt: number | null }> { + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + throw new StoreSchedulerBusyError( + 'queue_full', + 'background', + 'agent.finalizedSwmCleanup.backlog', + ); + } + const result = await this.store.query( + `SELECT (COUNT(DISTINCT ?task) AS ?count) (MIN(?markedAt) AS ?oldest) WHERE { + GRAPH <${assertSafeIri(swmMetaGraph)}> { + ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; + <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . + OPTIONAL { ?task <${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } + } + }`, + { + priority: 'background', + source: 'agent.finalizedSwmCleanup.backlog', + signal, + }, + ); + if (result.type !== 'bindings' || result.bindings.length === 0) { + return { depth: 0, oldestMarkerAt: null }; + } + const depth = Number.parseInt(stripOptionalLiteral(result.bindings[0]?.['count']) ?? '0', 10); + const oldestValue = stripOptionalLiteral(result.bindings[0]?.['oldest']); + const oldestMarkerAt = oldestValue ? Date.parse(oldestValue) : Number.NaN; + return { + depth: Number.isSafeInteger(depth) && depth > 0 ? depth : 0, + oldestMarkerAt: Number.isFinite(oldestMarkerAt) ? oldestMarkerAt : null, + }; + } + + private async cleanupMetaGraph(input: { + contextGraphId: string; + swmMetaGraph: string; + maxCandidates?: number; + signal?: AbortSignal; + }): Promise<{ deletedItems: number; examinedCandidates: number }> { + if (!this.writeLocks || hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + return { deletedItems: 0, examinedCandidates: 0 }; + } + const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); + const queryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup.discover', + signal: input.signal, + }; + const result = await this.store.query( + `SELECT DISTINCT ?task ?ual ?version ?root ?shareId ?assertionGraph ?headFingerprint ?subGraphName WHERE { + GRAPH <${assertSafeIri(input.swmMetaGraph)}> { + ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; + <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; + <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ?headFingerprint ; + ?ual ; + ?version ; + ?shareId ; + ?assertionGraph . + OPTIONAL { ?task ?subGraphName } + } + } ORDER BY ?task LIMIT ${limit}`, + queryOptions, + ); + if (result.type !== 'bindings') return { deletedItems: 0, examinedCandidates: 0 }; + + let cleared = 0; + for (const row of result.bindings) { + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) break; + const taskSubject = row['task']; + const ual = row['ual']; + const rawVersion = stripOptionalLiteral(row['version']); + const rawRoot = stripOptionalLiteral(row['root']); + const assertionGraph = row['assertionGraph']; + const expectedHeadFingerprint = stripOptionalLiteral(row['headFingerprint']); + const shareOperationId = stripOptionalLiteral(row['shareId']); + const subGraphName = stripOptionalLiteral(row['subGraphName']); + if ( + !taskSubject + || !taskSubject.startsWith('urn:dkg:finalized-swm-cleanup:') + || !ual + || !rawVersion + || !/^\d+$/.test(rawVersion) + || !rawRoot + || !assertionGraph + || !expectedHeadFingerprint + || !shareOperationId + ) continue; + + let scope: ReturnType; + let expectedMerkleRoot: Uint8Array; + try { + scope = createGraphKnowledgeAssetScope(ual, BigInt(rawVersion)); + expectedMerkleRoot = ethers.getBytes(rawRoot); + } catch { + continue; + } + if (scope.ual !== ual || expectedMerkleRoot.length !== 32) continue; + + let expectedHead: KnowledgeAssetWorkspaceHead | undefined; + try { + expectedHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager: new GraphManager(this.store), + contextGraphId: input.contextGraphId, + kaUal: scope.ual, + subGraphName, + queryOptions, + }); + } catch (error) { + if (error instanceof StoreSchedulerBusyError) break; + continue; + } + if ( + !expectedHead + || expectedHead.assertionVersion !== scope.assertionVersion + || expectedHead.shareOperationId !== shareOperationId + || expectedHead.assertionGraph !== assertionGraph + ) { + await this.retireStaleTask({ + contextGraphId: input.contextGraphId, + swmMetaGraph: input.swmMetaGraph, + taskSubject, + scope, + assertionGraph, + shareOperationId, + subGraphName, + queryOptions, + expectedHeadWasAbsent: !expectedHead, + }); + continue; + } + let privateMerkleRoot: Uint8Array | undefined; + try { + privateMerkleRoot = expectedHead.privateMerkleRoot + ? ethers.getBytes(expectedHead.privateMerkleRoot) + : undefined; + } catch { + continue; + } + const outcome = await this.clearIfStillExact({ + contextGraphId: input.contextGraphId, + scope, + taskSubject, + expectedHeadFingerprint, + expectedHead, + expectedMerkleRoot, + privateMerkleRoot, + subGraphName, + signal: input.signal, + }); + if (outcome === 'cleared') cleared += 1; + } + return { deletedItems: cleared, examinedCandidates: result.bindings.length }; + } + + private async retireStaleTask(input: { + contextGraphId: string; + swmMetaGraph: string; + taskSubject: string; + scope: ReturnType; + assertionGraph: string; + shareOperationId: string; + subGraphName?: string; + queryOptions: QueryOptions; + expectedHeadWasAbsent: boolean; + }): Promise { + if (!this.writeLocks) return; + const writePrefix = `${contextGraphDataUri(input.contextGraphId)}/`; + const preflightWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + let headlessPayloadAbsent = false; + if (input.expectedHeadWasAbsent) { + if (preflightWriteGen === undefined) return; + const payloadPresent = await this.store.query( + `ASK { GRAPH <${assertSafeIri(input.assertionGraph)}> { ?s ?p ?o } }`, + { + ...input.queryOptions, + source: 'agent.finalizedSwmCleanup.checkHeadlessPayload', + }, + ); + if (payloadPresent.type !== 'boolean' || payloadPresent.value) return; + headlessPayloadAbsent = true; + } + const lockKey = swmKaWriteLockKey( + input.contextGraphId, + input.subGraphName, + input.scope.ual, + ); + await withKeyedLocks(this.writeLocks, [lockKey], async () => { + if ( + preflightWriteGen !== undefined + && this.graphWriteGen?.getWriteGen(writePrefix) !== preflightWriteGen + ) return; + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager: new GraphManager(this.store), + contextGraphId: input.contextGraphId, + kaUal: input.scope.ual, + subGraphName: input.subGraphName, + queryOptions: input.queryOptions, + }); + } catch (error) { + if (error instanceof KnowledgeAssetWorkspaceHeadCorruptError) return; + throw error; + } + if ( + currentHead + && currentHead.assertionVersion === input.scope.assertionVersion + && currentHead.shareOperationId === input.shareOperationId + && currentHead.assertionGraph === input.assertionGraph + ) return; + if (!currentHead && !headlessPayloadAbsent) return; + + await this.store.deleteByPattern( + { graph: input.swmMetaGraph, subject: input.taskSubject }, + { + ...input.queryOptions, + source: currentHead + ? 'agent.finalizedSwmCleanup.retireSupersededTask' + : 'agent.finalizedSwmCleanup.retireAbsentTask', + }, + ); + }); + } + + private async clearIfStillExact(input: { + contextGraphId: string; + scope: ReturnType; + taskSubject: string; + expectedHeadFingerprint: string; + expectedHead: KnowledgeAssetWorkspaceHead; + expectedMerkleRoot: Uint8Array; + privateMerkleRoot?: Uint8Array; + subGraphName?: string; + signal?: AbortSignal; + }): Promise<'cleared' | 'absent' | 'preserved'> { + if (!this.writeLocks) return 'preserved'; + const graphManager = new GraphManager(this.store); + const metaGraph = graphManager.sharedMemoryMetaUri(input.contextGraphId, input.subGraphName); + const headSubject = workspaceKnowledgeAssetHeadSubject(input.scope.ual); + const cleanupRootObject = JSON.stringify( + ethers.hexlify(input.expectedMerkleRoot).toLowerCase(), + ); + const verificationQueryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup', + signal: input.signal, + }; + const commitQueryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup', + }; + if ( + finalizedSwmCleanupHeadFingerprint(input.expectedHead) + !== input.expectedHeadFingerprint + ) return 'preserved'; + + const writePrefix = `${contextGraphDataUri(input.contextGraphId)}/`; + const preflightWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) return 'preserved'; + const vmVerification = await verifyExactGraphScopedLayer({ + store: this.store, + contextGraphId: input.contextGraphId, + scope: input.scope, + layer: MemoryLayer.VerifiableMemory, + publicTripleCount: input.expectedHead.publicTripleCount, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + expectedPublicQuadsDigest: input.expectedHead.publicQuadsDigest, + subGraphName: input.subGraphName, + queryOptions: verificationQueryOptions, + }); + if (vmVerification.status !== 'verified') { + this.log.warn( + createOperationContext('system'), + `Preserving ${input.scope.ual}; VM no longer matches cleanup task (${vmVerification.status})`, + ); + return 'preserved'; + } + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) return 'preserved'; + const swmVerification = await verifyExactGraphScopedLayer({ + store: this.store, + contextGraphId: input.contextGraphId, + scope: input.scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: input.expectedHead.publicTripleCount, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + expectedPublicQuadsDigest: input.expectedHead.publicQuadsDigest, + subGraphName: input.subGraphName, + queryOptions: verificationQueryOptions, + }); + if ( + swmVerification.status !== 'verified' + && !(swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) + ) { + this.log.warn( + createOperationContext('system'), + `Preserving ${input.scope.ual}; SWM no longer matches cleanup task (${swmVerification.status})`, + ); + return 'preserved'; + } + const verifiedWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + if (preflightWriteGen !== undefined && verifiedWriteGen !== preflightWriteGen) { + return 'preserved'; + } + + const lockKey = swmKaWriteLockKey( + input.contextGraphId, + input.subGraphName, + input.scope.ual, + ); + const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { + if ( + verifiedWriteGen !== undefined + && this.graphWriteGen?.getWriteGen(writePrefix) !== verifiedWriteGen + ) return 'preserved' as const; + + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager, + contextGraphId: input.contextGraphId, + kaUal: input.scope.ual, + subGraphName: input.subGraphName, + queryOptions: commitQueryOptions, + }); + } catch (error) { + if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; + return 'preserved' as const; + } + if (!currentHead) { + if (swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) { + await this.store.deleteByPattern( + { graph: metaGraph, subject: input.taskSubject }, + { ...commitQueryOptions, source: 'agent.finalizedSwmCleanup.retireAbsentTask' }, + ); + return 'absent' as const; + } + return 'preserved' as const; + } + if ( + !sameKnowledgeAssetWorkspaceHead(currentHead, input.expectedHead) + || finalizedSwmCleanupHeadFingerprint(currentHead) !== input.expectedHeadFingerprint + ) return 'preserved' as const; + + const marker = await this.store.query( + `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` + + `<${assertSafeIri(input.taskSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` + + `${cleanupRootObject} ; <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ` + + `${JSON.stringify(input.expectedHeadFingerprint)} } }`, + commitQueryOptions, + ); + if (marker.type !== 'boolean' || !marker.value) return 'preserved' as const; + + const replaced = await tryReplaceGraphAndSubjectAtomically( + this.store, + swmVerification.graphUri, + [], + metaGraph, + headSubject, + [], + commitQueryOptions, + ); + if (!replaced) { + throw Object.assign( + new Error('Finalized SWM cleanup requires atomic graph-and-head replacement support'), + { code: 'SWM_ATOMIC_CLEANUP_UNSUPPORTED' }, + ); + } + await this.store.deleteByPattern( + { graph: metaGraph, subject: input.taskSubject }, + { ...commitQueryOptions, source: 'agent.finalizedSwmCleanup.retireTask' }, + ); + return swmVerification.status === 'verified' ? 'cleared' as const : 'absent' as const; + }); + + if (outcome === 'cleared') { + this.eventBus?.emit(DKGEvent.MEMORY_GRAPH_CHANGED, { + contextGraphId: input.contextGraphId, + layers: ['swm'], + subGraphName: input.subGraphName, + operation: 'shared_working_memory_finalized', + source: 'background-cleanup', + counts: { triples: input.expectedHead.publicTripleCount }, + }); + this.log.info( + createOperationContext('system'), + `Cleared finalized graph-scoped SWM assertion ${input.scope.ual}`, + ); + } + return outcome; + } +} diff --git a/packages/agent/src/graph-scoped-layer-verification.ts b/packages/agent/src/graph-scoped-layer-verification.ts new file mode 100644 index 0000000000..42adbf2677 --- /dev/null +++ b/packages/agent/src/graph-scoped-layer-verification.ts @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { + MemoryLayer, + assertSafeIri, + createGraphKnowledgeAssetScope, + knowledgeAssetLayerGraphUri, +} from '@origintrail-official/dkg-core'; +import type { Quad, QueryOptions, TripleStore } from '@origintrail-official/dkg-storage'; +import { + computeFlatKCRootV10 as computeFlatKCRoot, + type KnowledgeAssetWorkspaceHead, + workspacePublicQuadsDigest, +} from '@origintrail-official/dkg-publisher'; + +export type ExactGraphScopedLayerVerification = + | { + status: 'verified'; + graphUri: string; + quads: Quad[]; + merkleRoot: Uint8Array; + } + | { + status: 'count-mismatch'; + graphUri: string; + actualCount: number; + } + | { + status: 'merkle-mismatch'; + graphUri: string; + } + | { + status: 'head-mismatch'; + graphUri: string; + }; + +function equalBytes(left: Uint8Array, right: Uint8Array): boolean { + return left.length === right.length + && left.every((byte, index) => byte === right[index]); +} + +/** + * Verify one exact graph-scoped VM or SWM layer from live payload content. + * This helper deliberately has no cleanup policy and is shared by foreground + * finalization and the independent finalized-SWM maintenance component. + */ +export async function verifyExactGraphScopedLayer(input: { + store: TripleStore; + contextGraphId: string; + scope: ReturnType; + layer: MemoryLayer.SharedWorkingMemory | MemoryLayer.VerifiableMemory; + publicTripleCount: number; + privateMerkleRoot?: Uint8Array; + expectedMerkleRoot: Uint8Array; + expectedPublicQuadsDigest?: KnowledgeAssetWorkspaceHead['publicQuadsDigest']; + subGraphName?: string; + queryOptions?: QueryOptions; +}): Promise { + const graphUri = knowledgeAssetLayerGraphUri( + input.contextGraphId, + input.layer, + input.scope, + input.subGraphName, + ); + const result = await input.store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(graphUri)}> { ?s ?p ?o } }`, + input.queryOptions, + ); + const quads = result.type === 'quads' + ? result.quads.map((quad) => ({ ...quad, graph: '' })) + : []; + if (quads.length !== input.publicTripleCount) { + return { status: 'count-mismatch', graphUri, actualCount: quads.length }; + } + const merkleRoot = computeFlatKCRoot( + quads, + input.privateMerkleRoot ? [input.privateMerkleRoot] : [], + ); + if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { + return { status: 'merkle-mismatch', graphUri }; + } + if ( + input.expectedPublicQuadsDigest !== undefined + && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest + ) { + return { status: 'head-mismatch', graphUri }; + } + return { status: 'verified', graphUri, quads, merkleRoot }; +} diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 0b7a7fdcc9..5b360241ec 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -47,6 +47,7 @@ import { import { createCursorState } from '../src/reconcile-cursor.js'; import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped-swm-recovery.js'; import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; +import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; const CG = 'rootless-finalization'; const AUTHOR = '0x1111111111111111111111111111111111111111'; @@ -188,11 +189,22 @@ describe('graph-scoped finalization handler', () => { handler = new FinalizationHandler(store, legacyFinalizationChain(), { writeLocks }); }); + function cleanupService( + locks: Map> | null = writeLocks, + ): FinalizedSwmCleanupService { + return new FinalizedSwmCleanupService({ + store, + writeLocks: locks ?? undefined, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [graphManager.sharedMemoryMetaUri(CG)], + }); + } + async function drainFinalizedSwm( - target = handler, + locks: Map> | null = writeLocks, subGraphName?: string, ): Promise { - return target.cleanupFinalizedGraphScopedSwmWhenIdle({ + return cleanupService(locks).cleanupKnownMetaGraph({ contextGraphId: CG, swmMetaGraph: graphManager.sharedMemoryMetaUri(CG, subGraphName), maxCandidates: 16, @@ -2338,7 +2350,7 @@ describe('graph-scoped finalization handler', () => { await blocker; await finalization; expect(await store.countQuads(swmGraph)).toBe(2); - expect(await drainFinalizedSwm(lockingHandler)).toBe(1); + expect(await drainFinalizedSwm(writeLocks)).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); }); @@ -2351,7 +2363,7 @@ describe('graph-scoped finalization handler', () => { CG, ); - expect(await uncoordinated.cleanupFinalizedGraphScopedSwmWhenIdle({ + expect(await cleanupService(null).cleanupKnownMetaGraph({ contextGraphId: CG, swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), maxCandidates: 16, @@ -2389,10 +2401,7 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(2); busy = false; - const restarted = new FinalizationHandler(store, legacyFinalizationChain(), { - writeLocks, - }); - expect(await drainFinalizedSwm(restarted)).toBe(1); + expect(await drainFinalizedSwm()).toBe(1); expect(await store.countQuads(swmGraph)).toBe(0); await expect(resolveKnowledgeAssetWorkspaceHead({ store, @@ -2446,13 +2455,13 @@ describe('graph-scoped finalization handler', () => { const discoverQueries: string[] = []; const originalQuery = store.query.bind(store); const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { - if (options?.source === 'agent.finalization.graphScopedSwmCleanup.discover') { + if (options?.source === 'agent.finalizedSwmCleanup.discover') { discoverQueries.push(query); } return originalQuery(query, options); }); - await expect(handler.cleanupFinalizedGraphScopedSwmWhenIdle({ + await expect(cleanupService().cleanupKnownMetaGraph({ contextGraphId: CG, swmMetaGraph: graphManager.sharedMemoryMetaUri(CG, subGraphName), maxCandidates: 1, @@ -2479,12 +2488,12 @@ describe('graph-scoped finalization handler', () => { let injectedBusyTimeout = false; const cleanupPriorities: Array = []; const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { - if (options?.source?.startsWith('agent.finalization.graphScopedSwmCleanup')) { + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { cleanupPriorities.push(options.priority); } if ( !injectedBusyTimeout - && options?.source === 'agent.finalization.graphScopedSwmCleanup.discover' + && options?.source === 'agent.finalizedSwmCleanup.discover' && query.includes('SELECT DISTINCT ?task ?ual ?version ?root ?shareId') ) { injectedBusyTimeout = true; @@ -2497,7 +2506,7 @@ describe('graph-scoped finalization handler', () => { return originalQuery(query, options); }); - await expect(handler.cleanupFinalizedGraphScopedSwmWhenIdle({ + await expect(cleanupService().cleanupKnownMetaGraph({ contextGraphId: CG, swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), maxCandidates: 16, diff --git a/packages/agent/test/swm-ttl-v2-cleanup.test.ts b/packages/agent/test/swm-ttl-v2-cleanup.test.ts index 5fcc25d33d..54d450b004 100644 --- a/packages/agent/test/swm-ttl-v2-cleanup.test.ts +++ b/packages/agent/test/swm-ttl-v2-cleanup.test.ts @@ -239,17 +239,6 @@ describe('SWM TTL cleanup of graph-scoped V2 operations', () => { it('honors an explicit public owner/name context graph during finalized cleanup', async () => { const cg = '0x1111111111111111111111111111111111111111/public-finalized-cleanup'; - const cleanupFinalizedGraphScopedSwmWhenIdle = vi.fn().mockResolvedValue(0); - const inspectFinalizedGraphScopedSwmCleanupBacklog = vi.fn().mockResolvedValue({ - depth: 0, - oldestMarkerAt: null, - }); - const handlerSpy = vi - .spyOn(node as unknown as { getOrCreateFinalizationHandler: () => unknown }, 'getOrCreateFinalizationHandler') - .mockReturnValue({ - cleanupFinalizedGraphScopedSwmWhenIdle, - inspectFinalizedGraphScopedSwmCleanupBacklog, - }); const listSpy = vi.spyOn(node, 'listContextGraphs').mockResolvedValue([{ id: cg, uri: `did:dkg:context-graph:${cg}`, @@ -258,18 +247,24 @@ describe('SWM TTL cleanup of graph-scoped V2 operations', () => { subscribed: true, synced: true, }]); + const originalQuery = store.query.bind(store); + const discoveryQueries: string[] = []; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.discover') { + discoveryQueries.push(query); + } + return originalQuery(query, options); + }); try { await node.runFinalizedSwmCleanupSweep(); } finally { - handlerSpy.mockRestore(); + querySpy.mockRestore(); listSpy.mockRestore(); } - expect(cleanupFinalizedGraphScopedSwmWhenIdle).toHaveBeenCalledWith(expect.objectContaining({ - contextGraphId: cg, - swmMetaGraph: contextGraphSharedMemoryMetaUri(cg), - maxCandidates: 1, - })); + expect(discoveryQueries.some((query) => query.includes( + `GRAPH <${contextGraphSharedMemoryMetaUri(cg)}>`, + ))).toBe(true); }); }); From 22f6f04e071850cf7ec5ded58ed25654de564cff Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 09:44:17 +0200 Subject: [PATCH 15/48] test(agent): pin the finalized-SWM GC safety boundaries The idle finalized-SWM garbage collector had three unguarded behaviours: its in-lock head re-check, its mid-sweep abort, and its wall-clock budget could all be neutralized with the suite still fully green. Adds two TOCTOU tests around the commit path. Discovery and both payload verifications deliberately run outside the per-KA writer lock, so the head read taken under the lock is the only proof that the assertion being deleted is still the one that was verified. Mutating the head before the drain does not reach that branch (pre-lock triage retires the task first), so both tests land the race between the two reads: one through the commit head read, one by contending the writer lock on a store whose write-generation capability is unavailable, where the re-check is provably the sole remaining guard. Adds a sweep-gate suite covering the reviewer's acceptance criterion that the GC performs no discovery or payload scan under sustained load, asserted against the store itself rather than the discovery closure, plus each in-loop pressure and deadline return and a budget-exhausted slice driven through the worker by the real service. Every assertion is about work NOT attempted; asserting only the returned sweep result passes with the gates deleted, because the callee-side guards produce the same result shape one step later. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/finalized-swm-cleanup-sweep.test.ts | 390 ++++++++++++++++++ .../ka-graph-finalization-handler.test.ts | 150 +++++++ packages/agent/vitest.unit.config.ts | 1 + 3 files changed, 541 insertions(+) create mode 100644 packages/agent/test/finalized-swm-cleanup-sweep.test.ts diff --git a/packages/agent/test/finalized-swm-cleanup-sweep.test.ts b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts new file mode 100644 index 0000000000..46df8742be --- /dev/null +++ b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Control-flow guards of the idle finalized-SWM GC slice. + * + * `FinalizedSwmCleanupService.runSweep()` is the only discovery boundary the + * garbage collector owns, and the reviewer's acceptance criteria are about what + * it *refuses* to do: under sustained foreground load it must perform no + * discovery and no payload scan at all, and once its wall-clock budget is spent + * it must yield the slice rather than keep enumerating. + * + * Those are properties of the in-loop `return`s, not of the happy path, so the + * assertions here are deliberately about work NOT attempted — every test pins a + * specific gate by observing the next store read or discovery closure that would + * run if the gate were removed. A test that only asserted the returned + * `FinalizedSwmCleanupSweepResult` would pass with the gates deleted, because + * the callee-side guards inside `cleanupMetaGraph`/`inspectBacklog` produce the + * same result shape one step later. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { GraphManager, OxigraphStore, type TripleStore } from '@origintrail-official/dkg-storage'; +import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; +import { FinalizedSwmCleanupWorker } from '../src/finalized-swm-cleanup-worker.js'; + +const CG = 'sweep-gate-cg'; + +/** Toggleable foreground pressure; the service only reads the snapshot. */ +function installPressureSwitch(store: TripleStore): { busy: boolean } { + const state = { busy: false }; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: state.busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + return state; +} + +/** + * Records every store *method call* (not property read) so a test can assert + * that a pass touched the store zero times. `asGraphWriteGenSource` only probes + * `typeof store.getWriteGen === 'function'`, which stays a property read here. + */ +function countingStore(inner: TripleStore, calls: string[]): TripleStore { + return new Proxy(inner as unknown as object, { + get(target, prop) { + const value = Reflect.get(target, prop); + if (typeof value !== 'function') return value; + return (...args: unknown[]) => { + calls.push(String(prop)); + return (value as (...a: unknown[]) => unknown).apply(target, args); + }; + }, + }) as unknown as TripleStore; +} + +/** Collects the `source` label of every query the sweep issues. */ +function recordQuerySources(store: OxigraphStore): { + sources: string[]; + restore: () => void; +} { + const sources: string[] = []; + const originalQuery = store.query.bind(store); + const spy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + sources.push(options.source); + } + return originalQuery(query, options); + }); + return { sources, restore: () => spy.mockRestore() }; +} + +describe('finalized SWM cleanup sweep gates', () => { + /** + * Reviewer acceptance bullet (a): under sustained load the GC performs no + * discovery and no payload scan. Asserting only that the discovery closure + * was skipped is weaker than the bullet — it leaves room for a refactor to + * hoist a store read (a backlog probe, a resumption-cursor read) above the + * pressure gate. This pins the store itself: the only call permitted while + * pressure is active is reading the pressure snapshot. + */ + it('performs zero store operations and no discovery while the store is under pressure', async () => { + const inner = new OxigraphStore(); + const pressure = installPressureSwitch(inner); + pressure.busy = true; + const calls: string[] = []; + const listContextGraphIds = vi.fn(async () => [CG]); + const listSharedMemoryMetaGraphs = vi.fn(async () => [ + new GraphManager(inner).sharedMemoryMetaUri(CG), + ]); + const service = new FinalizedSwmCleanupService({ + store: countingStore(inner, calls), + writeLocks: new Map>(), + listContextGraphIds, + listSharedMemoryMetaGraphs, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + backlogDepth: 0, + }); + + expect(listContextGraphIds).not.toHaveBeenCalled(); + expect(listSharedMemoryMetaGraphs).not.toHaveBeenCalled(); + expect(calls.filter((name) => name !== 'getPressureSnapshot')).toEqual([]); + expect(calls.length).toBeGreaterThan(0); + }); + + /** + * Pressure that appears *after* the context-graph enumeration resolves — the + * realistic case, since enumeration is the expensive call that tends to run + * just as foreground work arrives. The gate at the top of the context-graph + * loop must abandon the slice before the per-graph meta enumeration. + */ + it('abandons the slice when pressure appears after context-graph enumeration', async () => { + const store = new OxigraphStore(); + const pressure = installPressureSwitch(store); + const listSharedMemoryMetaGraphs = vi.fn(async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ]); + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => { + pressure.busy = true; + return [CG, 'second-cg']; + }, + listSharedMemoryMetaGraphs, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + + expect(listSharedMemoryMetaGraphs).not.toHaveBeenCalled(); + expect(sources).toEqual([]); + restore(); + }); + + /** + * Pressure that appears while a meta graph is being cleaned. The gate after + * `cleanupMetaGraph` must abandon the slice: without it the backlog probe + * runs and throws `StoreSchedulerBusyError` into the worker, converting an + * expected, retryable idle-yield into a recorded sweep failure. + */ + it('abandons the slice when pressure appears during meta-graph cleanup', async () => { + const store = new OxigraphStore(); + const pressure = installPressureSwitch(store); + const sources: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + const result = await originalQuery(query, options); + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + sources.push(options.source); + if (options.source === 'agent.finalizedSwmCleanup.discover') pressure.busy = true; + } + return result; + }); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + + expect(sources).toEqual(['agent.finalizedSwmCleanup.discover']); + expect(sources).not.toContain('agent.finalizedSwmCleanup.backlog'); + querySpy.mockRestore(); + }); + + /** + * A pressure snapshot is a point-in-time sample: a foreground burst can be + * over by the time a callee re-samples it. The gate at the top of the + * meta-graph loop is therefore not redundant with the guard inside + * `cleanupMetaGraph` — it is the only observer of a spike seen at that + * instant, and dropping it lets a slice that already saw pressure go on to + * open a discovery scan. + * + * The spike is one-shot on purpose. Adding a further gate before this one + * keeps the test green (the slice is still abandoned, still with no scan); + * only removing the gate turns it red. + */ + it('abandons the slice on a transient pressure spike seen after meta-graph enumeration', async () => { + const store = new OxigraphStore(); + let spikePending = false; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => { + const busy = spikePending; + spikePending = false; + return { + ackInflight: 0, + healthInflight: 0, + normalInflight: busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }; + }, + }); + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => { + spikePending = true; + return [new GraphManager(store).sharedMemoryMetaUri(CG)]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + + expect(sources).toEqual([]); + restore(); + }); + + /** + * The wall-clock budget is enforced on a caller-supplied clock, so these + * three tests move the clock at each in-loop boundary. `budgetExhausted` is + * what makes the worker reschedule promptly instead of waiting for the + * periodic backstop, so both the flag and the abandoned work are asserted. + */ + it('yields the slice when the budget expires during context-graph enumeration', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const listSharedMemoryMetaGraphs = vi.fn(async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ]); + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => { + clock.now += 60_001; + return [CG, 'second-cg']; + }, + listSharedMemoryMetaGraphs, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + budgetExhausted: true, + deletedItems: 0, + }); + + expect(listSharedMemoryMetaGraphs).not.toHaveBeenCalled(); + expect(sources).toEqual([]); + restore(); + }); + + it('yields the slice when the budget expires during meta-graph enumeration', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => { + clock.now += 60_001; + return [new GraphManager(store).sharedMemoryMetaUri(CG)]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + budgetExhausted: true, + deletedItems: 0, + }); + + // Neither the candidate discovery nor the backlog probe may run once the + // slice is over budget. + expect(sources).toEqual([]); + restore(); + }); + + it('yields the slice when the budget expires inside meta-graph cleanup', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const sources: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + const result = await originalQuery(query, options); + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + sources.push(options.source); + if (options.source === 'agent.finalizedSwmCleanup.discover') clock.now += 60_001; + } + return result; + }); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + budgetExhausted: true, + deletedItems: 0, + }); + + expect(sources).toEqual(['agent.finalizedSwmCleanup.discover']); + expect(sources).not.toContain('agent.finalizedSwmCleanup.backlog'); + querySpy.mockRestore(); + }); + + /** + * End-to-end budget wiring: the worker's prompt-retry decision is driven by a + * `budgetExhausted` produced by the real service, not by a stubbed sweep, and + * a yielded slice must not be miscounted as store pressure. + */ + it('reschedules promptly on a budget-exhausted slice produced by the real service', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => { + clock.now += 60_001; + return [CG]; + }, + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + const timers: Array<{ fn: () => void; delayMs: number }> = []; + const onError = vi.fn(); + const worker = new FinalizedSwmCleanupWorker({ + sweep: () => service.runSweep(), + retryDelayMs: 321, + onError, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + + await worker.runNow(); + + expect(onError).not.toHaveBeenCalled(); + expect(worker.snapshot()).toMatchObject({ runs: 1, pressureSkips: 0, lastError: null }); + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(321); + await worker.close(); + }); +}); diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 5b360241ec..110aa216de 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -15,7 +15,9 @@ import { GraphManager, OxigraphStore, StoreSchedulerBusyError, + asGraphWriteGenSource, type Quad, + type TripleStore, } from '@origintrail-official/dkg-storage'; import type { ChainAdapter } from '@origintrail-official/dkg-chain'; import { @@ -135,6 +137,33 @@ function legacyFinalizationChain( } as ChainAdapter; } +/** The read the cleanup service takes again once it owns the writer lock. */ +const WORKSPACE_HEAD_READ_PREFIX = 'SELECT ?scopeVersion ?kaUal ?assertionVersion'; + +const WRITE_GEN_CAPABILITY_KEYS = new Set(['getWriteGen', 'innerStore', 'inner']); + +/** + * A store whose per-graph write-generation capability cannot be recovered. + * + * `asGraphWriteGenSource` is documented as fail-open, so on such a store every + * generation comparison in the cleanup service is inert and the head re-read + * taken inside the writer lock is the only remaining proof that nothing moved + * while verification ran outside it. + */ +function withoutWriteGenTracking(inner: OxigraphStore): TripleStore { + return new Proxy(inner as unknown as object, { + get(target, prop) { + if (WRITE_GEN_CAPABILITY_KEYS.has(prop as string)) return undefined; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + has(target, prop) { + if (WRITE_GEN_CAPABILITY_KEYS.has(prop as string)) return false; + return Reflect.has(target, prop); + }, + }) as unknown as TripleStore; +} + async function closeInbox(inbox: SqliteFinalizationRecoveryStore | undefined): Promise { await inbox?.close().catch(() => {}); } @@ -2354,6 +2383,127 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(0); }); + it('preserves an assertion replaced between the pre-lock head read and the in-lock re-check', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await store.countQuads(swmGraph)).toBe(2); + + // Discovery and both payload verifications run OUTSIDE the per-KA writer + // lock by design, so everything the drain has proven is only as fresh as + // the head it read before taking the lock. The re-read taken under the + // lock is the sole check that closes that window: the write-generation + // comparisons are already behind us by then, and the marker ASK only + // proves the cleanup task still exists, not that the assertion is still + // the one that was verified. + // + // Mutating the head *before* the drain does not reach this branch — the + // pre-lock triage in `cleanupMetaGraph` sees the mismatch first and + // retires the task. The race has to land between the two reads. + const originalQuery = store.query.bind(store); + let racedInsideLock = false; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if ( + !racedInsideLock + && options?.source === 'agent.finalizedSwmCleanup' + && query.startsWith(WORKSPACE_HEAD_READ_PREFIX) + ) { + racedInsideLock = true; + // Drop the seam first so the staging writes and the rest of the commit + // path run against the real store. + querySpy.mockRestore(); + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + } + return originalQuery(query, options); + }); + + await expect(drainFinalizedSwm()).resolves.toBe(0); + expect(racedInsideLock).toBe(true); + + // The newer assertion shares the SWM graph URI with the finalized one, so + // a delete here destroys unpublished data rather than a redundant copy. + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(store.query( + `ASK { GRAPH <${swmGraph}> { ` + + ` "newer" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '2' }); + }); + + it('re-reads the head under the lock when the store cannot track write generations', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await store.countQuads(swmGraph)).toBe(2); + + const untrackedStore = withoutWriteGenTracking(store); + expect(asGraphWriteGenSource(untrackedStore)).toBeNull(); + + const lockKey = swmKaWriteLockKey(CG, undefined, UAL); + let releaseWriter!: () => void; + let markWriterHolding!: () => void; + const writerHolding = new Promise((resolve) => { markWriterHolding = resolve; }); + const writerDone = new Promise((resolve) => { releaseWriter = resolve; }); + const blocker = withKeyedLocks(writeLocks, [lockKey], async () => { + markWriterHolding(); + await writerDone; + }); + // `withKeyedLocks` installs its gate synchronously, before its first await. + const writerGate = writeLocks.get(lockKey); + expect(writerGate).toBeDefined(); + await writerHolding; + + const drain = new FinalizedSwmCleanupService({ + store: untrackedStore, + writeLocks, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [graphManager.sharedMemoryMetaUri(CG)], + }).cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + }); + + // Wait until the drain has finished discovery plus both verifications and + // queued behind the writer; the map entry flips to the drain's own gate. + let drainQueuedBehindWriter = false; + for (let attempt = 0; attempt < 1_000 && !drainQueuedBehindWriter; attempt += 1) { + await new Promise((resolve) => { setImmediate(resolve); }); + drainQueuedBehindWriter = writeLocks.get(lockKey) !== writerGate; + } + expect(drainQueuedBehindWriter).toBe(true); + + // The lock holder replaces the assertion the drain just verified. Nothing + // the drain captured before queueing is true any more. + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + releaseWriter(); + await blocker; + + await expect(drain).resolves.toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(store.query( + `ASK { GRAPH <${swmGraph}> { ` + + ` "newer" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '2' }); + }); + it('preserves finalized SWM when the handler has no shared writer lock', async () => { const { message, swmGraph } = await stageGraph(); const uncoordinated = new FinalizationHandler(store, legacyFinalizationChain()); diff --git a/packages/agent/vitest.unit.config.ts b/packages/agent/vitest.unit.config.ts index ddcee0f96e..130e32f77d 100644 --- a/packages/agent/vitest.unit.config.ts +++ b/packages/agent/vitest.unit.config.ts @@ -116,6 +116,7 @@ export default defineConfig({ "test/startup-jitter.test.ts", "test/finalization-lifecycle-logger.test.ts", "test/finalized-swm-cleanup-worker.test.ts", + "test/finalized-swm-cleanup-sweep.test.ts", "test/finalization-handler.test.ts", "test/finalization-handler-chain-truth.test.ts", "test/finalization-handler-defensive-cg-id.test.ts", From c2deec072183cf14b32384e5e1fa5988a59ad7ed Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 09:45:23 +0200 Subject: [PATCH 16/48] perf(agent): keep local GC rows off the peer-serving snapshot budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readSwmMetaGraphSnapshot read `?s ?p ?o` unfiltered and accumulated every row against SYNC_RESPONDER_SNAPSHOT_BUILD_MAX_ROWS/_BYTES, stripping the finalized-cleanup rows only afterwards in process. Purely local GC bookkeeping therefore counted toward the ceiling that decides whether a context graph can be served at all — the #1847/#1868 snapshotBudgetError class. Apply the same store-side filter the fresh/TTL lanes already use. filterSwmMetaSnapshotRows also did its whole O(rows) index build before the early returns that used to fire first, so an unparseable cutoff paid for a full scan and then returned nothing. Settle that case up front, and build the per-subject map only in the TTL lane that consumes it. The cleanup-row strip deliberately stays above the TTL-disabled return: that lane must be filtered too, or the responder advertises local GC metadata. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../agent/src/sync/responder/graph-plan.ts | 61 ++++++++++++------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 0a94ce3420..8b848af0ec 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -57,6 +57,12 @@ const DKG_FINALIZED_SWM_CLEANUP_ROOT = `${DKG}finalizedSwmCleanupRoot`; const DKG_FINALIZED_SWM_CLEANUP_MARKED_AT = `${DKG}finalizedSwmCleanupMarkedAt`; const DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT = `${DKG}finalizedSwmCleanupHeadFingerprint`; const DKG_FINALIZED_SWM_CLEANUP_TASK = `${DKG}FinalizedSwmCleanupTask`; +/** In-process mirror of the store-side predicate filter below. */ +const LOCAL_FINALIZED_SWM_CLEANUP_PREDICATES = new Set([ + DKG_FINALIZED_SWM_CLEANUP_ROOT, + DKG_FINALIZED_SWM_CLEANUP_MARKED_AT, + DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT, +]); const localFinalizedSwmCleanupRowFilter = (subject: string, predicate: string): string => ` FILTER(${predicate} NOT IN ( <${DKG_FINALIZED_SWM_CLEANUP_ROOT}>, @@ -2612,9 +2618,18 @@ async function readBoundedSwmMetaSnapshot( } let result; try { + // Filter local finalized-cleanup rows AT THE STORE, like the fresh/TTL + // lanes already do. Stripping them in-process afterwards is not enough: + // every such row would first be accumulated against the peer-serving + // snapshot budget, so purely local GC bookkeeping could push a large + // context graph over SYNC_RESPONDER_SNAPSHOT_BUILD_MAX_ROWS/_BYTES and + // make it unsyncable (#1847/#1868 snapshotBudgetError). result = await store.query(` SELECT ?s ?p ?o WHERE { - GRAPH <${assertSafeIri(graph)}> { ?s ?p ?o } + GRAPH <${assertSafeIri(graph)}> { + ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} + } } LIMIT ${remainingRows + 1} `, { @@ -2671,6 +2686,30 @@ function filterSwmMetaSnapshotRows( rows: readonly SyncRow[], cutoffIso: string | null, ): SyncRow[] { + // An unparseable cutoff admits nothing, so settle it BEFORE any O(rows) + // work — otherwise the responder indexes and scans the whole snapshot only + // to throw it away. + const cutoffMs = cutoffIso == null ? Number.NaN : Date.parse(cutoffIso); + if (cutoffIso != null && !Number.isFinite(cutoffMs)) return []; + + // Finalized-SWM markers/tasks are local maintenance state. Strip only those + // rows; the active SWM lifecycle and its payload remain syncable until the + // independent idle GC safely removes them. This must stay ABOVE the + // TTL-disabled return: that lane is served from this function too, and an + // unfiltered return there advertises local GC metadata to peers. + const cleanupTaskSubjects = new Set(); + for (const row of rows) { + if (row.p === DKG_ONTOLOGY.RDF_TYPE && row.o === DKG_FINALIZED_SWM_CLEANUP_TASK) { + cleanupTaskSubjects.add(row.s); + } + } + const syncableRows = rows.filter((row) => + !cleanupTaskSubjects.has(row.s) && !LOCAL_FINALIZED_SWM_CLEANUP_PREDICATES.has(row.p)); + if (cutoffIso == null) return syncableRows.sort(compareRows); + + // TTL lane only: the per-subject index exists solely to answer the freshness + // and tuple-key questions below, so it is built after the lanes that cannot + // use it have already returned. const bySubject = new Map(); for (const row of rows) { const bucket = bySubject.get(row.s) ?? []; @@ -2681,7 +2720,6 @@ function filterSwmMetaSnapshotRows( (bySubject.get(subject) ?? []) .filter((row) => row.p === predicate) .map((row) => row.o); - const cutoffMs = cutoffIso == null ? Number.NaN : Date.parse(cutoffIso); const isFresh = (subject: string): boolean => objects(subject, DKG_PUBLISHED_AT) .some((value) => { const timestamp = Date.parse(stripLiteral(value)); @@ -2703,25 +2741,6 @@ function filterSwmMetaSnapshotRows( } return keys; }; - // Finalized-SWM markers/tasks are local maintenance state. Strip only those - // rows; the active SWM lifecycle and its payload remain syncable until the - // independent idle GC safely removes them. - const cleanupTaskSubjects = new Set(); - for (const [subject] of bySubject) { - if (objects(subject, DKG_ONTOLOGY.RDF_TYPE).includes(DKG_FINALIZED_SWM_CLEANUP_TASK)) { - cleanupTaskSubjects.add(subject); - } - } - const localCleanupPredicates = new Set([ - DKG_FINALIZED_SWM_CLEANUP_ROOT, - DKG_FINALIZED_SWM_CLEANUP_MARKED_AT, - DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT, - ]); - const syncableRows = rows.filter((row) => - !cleanupTaskSubjects.has(row.s) && !localCleanupPredicates.has(row.p)); - if (cutoffIso == null) return syncableRows.sort(compareRows); - if (!Number.isFinite(cutoffMs)) return []; - const admitted = new Set(); const freshOperationKeys = new Set(); for (const [subject] of bySubject) { From e96ca2903348104dfd81e0dd900080e2236df38d Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 09:45:39 +0200 Subject: [PATCH 17/48] perf(agent): read the finalization tombstone once per ingested KA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureFinalizedCleanupTask ran for every descriptor inside withKaWriteLock before any decision was taken, issuing a bound-subject CONSTRUCT even when no finalization had ever happened; replaceGraphScopedSwmHeadMetadata then re-ran the byte-identical query under the same lock. Two lock-held reads per KA, on the path review point 6 asked to keep O(1). Fold the re-arm into replaceGraphScopedSwmHeadMetadata, which already reads the tombstone and threw the derived task away. That call is the one that can resurrect a finalized SWM copy, so re-arming there is both the cheapest and the latest safe moment. The task subject is never in the delete set, so the insert is an idempotent re-arm, and the read now sits immediately before the delete instead of ahead of every decision — which narrows, though it cannot close, the window where a concurrently written tombstone is destroyed (FinalizationHandler writes markers without taking the per-KA SWM writer lock; its own writeLocks field is dead). ensureFinalizedCleanupTask is then called only on the two paths that never reach replaceHeadMetadata, and gates its CONSTRUCT behind an ASK over the identical bound pattern. Since the CONSTRUCT's WHERE requires a ?root binding, ASK-false is exactly the case the reader already discards — a strictly weaker query, not a policy gate, so no finalized marker can be missed. Tombstone reads per KA, measured on a real OxigraphStore: [no-marker] path 1/2b: 1 ASK (was 1 CONSTRUCT) [no-marker] path 2a/3: 1 CONSTRUCT (was 2 CONSTRUCT) [marker] path 2a/3: 1 CONSTRUCT (was 2 CONSTRUCT) [marker] path 1/2b: 1 ASK + 1 CONSTRUCT (was 1 CONSTRUCT) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/sync/requester/shared-memory-sync.ts | 20 ++++-- .../requester/swm-snapshot-materializer.ts | 65 +++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/sync/requester/shared-memory-sync.ts b/packages/agent/src/sync/requester/shared-memory-sync.ts index 2fc92843bf..81313e9ab2 100644 --- a/packages/agent/src/sync/requester/shared-memory-sync.ts +++ b/packages/agent/src/sync/requester/shared-memory-sync.ts @@ -317,11 +317,14 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // re-checks below stop the other failure the lock alone cannot: // replacing newer content with an older verified snapshot. // - // Re-arm only constant-size durable maintenance metadata when - // an earlier finalized cleanup left an operation tombstone. - // The independent GC owns all discovery, graph verification + // Every path below must leave a constant-size GC task armed + // when an earlier finalized cleanup left an operation + // tombstone, or a resurrected SWM copy would never be + // collected. `replaceHeadMetadata` does that from the tombstone + // read it already performs, so only the paths that never reach + // it call `ensureFinalizedCleanupTask` explicitly. The + // independent GC still owns all discovery, graph verification // and deletion; sync neither performs nor awaits that work. - await snapshotMaterializer.ensureFinalizedCleanupTask(pid, descriptor); // // (a) Version ordering. A stored head newer than the descriptor // means gossip advanced this KA past our snapshot; replacing @@ -336,6 +339,10 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro storedHead.version !== null && storedVersionOutranksDescriptor(storedHead.version, descriptor.assertionVersion) ) { + // Sync writes nothing here, but a previous round may have + // resurrected this KA and the GC may since have retired its + // task, so the tombstone still has to re-arm one. + await snapshotMaterializer.ensureFinalizedCleanupTask(pid, descriptor); for (const quad of descriptor.metadataQuads) { replacedGraphScopedMetaKeys.add(quadKey(quad)); } @@ -356,7 +363,12 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // version/operation rows) — e.g. a prior round replaced the // graph and then failed before finishing the metadata swap. // Install the verified head while the writer lock is held. + // This also re-arms any finalized-cleanup task. await snapshotMaterializer.replaceHeadMetadata(pid, descriptor); + } else { + // Clean head, content already present: nothing is written, + // so re-arm the GC task on its own. + await snapshotMaterializer.ensureFinalizedCleanupTask(pid, descriptor); } // Never append graph-scoped metadata again after releasing // this lock. For a clean head it is redundant; for a newer diff --git a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts index f709197086..2b9d716987 100644 --- a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts +++ b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts @@ -96,6 +96,11 @@ export interface SharedMemorySnapshotMaterializer { * Re-arm a constant-size GC task when this exact operation carries a local * finalization tombstone. This is marker bookkeeping only: no graph scan, * VM verification, deletion, or GC wait occurs on the ingest path. + * + * Call this ONLY on the decision paths that do not go on to + * {@link replaceHeadMetadata} — that replacement re-arms the task from the + * tombstone read it already performs, so calling both duplicates a + * lock-held store read per KA for no added guarantee. */ ensureFinalizedCleanupTask( contextGraphId: string, @@ -115,6 +120,11 @@ export interface SharedMemorySnapshotMaterializer { * `replaceMetaForGraphAssets`: without it the append-style meta insert stacks * old and new head rows on one subject and the durable current head becomes * ambiguous. + * + * Also re-arms the finalized-SWM GC task for this operation: this call is + * what can resurrect a finalized copy, and the tombstone read it performs + * anyway answers that too. Callers therefore need no separate + * {@link ensureFinalizedCleanupTask} on any path that reaches this. */ replaceHeadMetadata( contextGraphId: string, @@ -124,7 +134,7 @@ export interface SharedMemorySnapshotMaterializer { /** * Replace one graph-scoped SWM lifecycle without losing its immutable local - * finalization tombstone. + * finalization tombstone — and re-arm the GC task from the same read. * * Finalization markers are deliberately local-only and responders filter them * from synchronized metadata. A blind head/operation replacement therefore @@ -133,9 +143,19 @@ export interface SharedMemorySnapshotMaterializer { * subject. The independent GC task is stored on its own subject and is not * touched by this replacement. * - * The verified replacement metadata and any retained operation tombstone are - * inserted in the same store call after the old lifecycle is removed. Callers - * MUST hold the canonical per-KA SWM writer lock across this entire operation. + * Re-arming HERE rather than ahead of the caller's decisions is what keeps the + * ingest path at one tombstone read: this replacement is itself the act that + * can resurrect a finalized SWM copy, and the read it already performs answers + * both questions. Reading at the last possible moment before the delete also + * narrows — it cannot close — the window in which a concurrently written + * tombstone is destroyed, because `FinalizationHandler` writes markers without + * taking the per-KA SWM writer lock. + * + * The verified replacement metadata, any retained operation tombstone, and the + * constant-size GC task are inserted in the same store call after the old + * lifecycle is removed. The task's own subject is never in the delete set, so + * the insert is an idempotent re-arm. Callers MUST hold the canonical per-KA + * SWM writer lock across this entire operation. */ export async function replaceGraphScopedSwmHeadMetadata(params: { store: TripleStore; @@ -197,12 +217,42 @@ export async function replaceGraphScopedSwmHeadMetadata(params: { const replacementQuads = [ ...descriptor.metadataQuads, ...preserved.tombstone, + ...preserved.cleanupTask, ]; if (replacementQuads.length > 0) { await insertReplacementMetadata(replacementQuads); } } +/** + * Cheap existence gate for {@link readExactFinalizedOperationTombstone}. + * + * This is the same bound (graph, subject, predicate) pattern the CONSTRUCT + * below matches on, reduced to an ASK: no `?root` binding means that CONSTRUCT + * has no solutions, returns no quads, and the reader already gives up on + * `roots.length !== 1`. So a false here is exactly the reader's empty result — + * a strictly weaker query over the identical pattern, not a policy gate, and + * therefore incapable of missing a finalized marker. It is a live store read, + * so nothing about it can go stale, and nothing the GC reads depends on it. + */ +async function hasFinalizedOperationTombstone(params: { + store: TripleStore; + descriptor: GraphScopedSwmRecoveryDescriptor; + sourcePrefix: string; +}): Promise { + const { store, descriptor, sourcePrefix } = params; + const result = await store.query( + `ASK { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + { + priority: 'background', + source: `${sourcePrefix}.askFinalizedOperationTombstone`, + }, + ); + return result.type === 'boolean' && result.value; +} + async function readExactFinalizedOperationTombstone(params: { store: TripleStore; contextGraphId: string; @@ -348,6 +398,13 @@ export function createSharedMemorySnapshotMaterializer(deps: { }, ensureFinalizedCleanupTask: async (contextGraphId, descriptor) => { + // Most descriptors were never finalized, so answer that with one bound + // ASK instead of materializing a CONSTRUCT result per KA. + if (!await hasFinalizedOperationTombstone({ + store: deps.store, + descriptor, + sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + })) return; const preserved = await readExactFinalizedOperationTombstone({ store: deps.store, contextGraphId, From 5df21041d669b5777019b164a0b4f22c1559c438 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 09:45:57 +0200 Subject: [PATCH 18/48] perf(agent): bound and attribute late-receipt snapshot verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the idle GC has drained a finalized SWM copy, the immutable-snapshot fallback is the NORMAL path for a late receipt rather than an exception, so its cost has to be bounded and not merely correct. Discovery is a bounded metadata read already; the expensive half is per candidate — a full snapshot read plus digest plus Merkle root — and it ran once per discovered row, up to 16 times on one receipt. The digest filter does not save it: VerifiedGraphScopedFinalizationEvidence carries publicQuadsDigest as OPTIONAL, and with no digest only the triple count discriminates, so every candidate does full payload work. Cap the verifications at MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS and log truncation. The cap is inert when the digest is known, because candidates are then content-identical and the first match wins. Keep the receipt lane's default priority: receipts are latency-sensitive and demoting them to the background lane trades a CPU spike for receipt starvation under load. Instead make the work attributable — resolveKnowledgeAssetOperationPublicQuads accepted no query options at all, so its reads ran with no source and were invisible to the scheduler observability landed in #2003. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/finalization-handler.ts | 42 ++++++++++++++++++- .../publisher/src/workspace-resolution.ts | 17 +++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index a97e51a454..b20a9fa3e7 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -211,6 +211,18 @@ type GraphScopedMaterializationEnvelope = Pick< /** Immutable queued assertion envelope supplied only after receipt/seal validation. */ type TrustedGraphScopedAssertionEvidence = VerifiedGraphScopedFinalizationEvidence; +/** + * How many immutable operation snapshots one receipt may verify in full. + * + * Discovery is allowed to surface more candidates than this — it is a cheap + * bounded metadata read — but each verification costs a whole payload read + * plus digest plus Merkle root, on the latency-sensitive receipt path. Bound + * the expensive half so that a KA re-shared many times cannot turn one late + * receipt into a multiple of that work. See + * `FinalizationHandler.verifyImmutableGraphScopedSnapshot`. + */ +const MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS = 4; + function resolveGraphScopedAccessEnvelope( head: GraphScopedMaterializationEnvelope, requestedAccessPolicy?: GraphScopedAccessPolicy, @@ -1362,6 +1374,25 @@ export class FinalizationHandler { * Rebuild a finalized source from its immutable operation snapshot when the * active SWM graph has already been drained. This keeps durable receipt/reorg * recovery independent from retaining a second live copy of the payload. + * + * Once the idle GC has done its job this is the NORMAL path for a late + * receipt, not an exception, so its cost has to be bounded rather than + * merely correct. Two rules hold it down: + * + * - Discovery stays a bounded metadata read (one bound `kaUal`, LIMIT 16) + * and keeps the receipt lane's default priority. Receipts are + * latency-sensitive; demoting them to the background lane would trade a + * CPU spike for receipt starvation under load, which is the worse failure. + * - Payload verification is the expensive half — a full snapshot read plus + * digest plus Merkle root PER CANDIDATE — so it is capped at + * {@link MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS} instead of running once per + * discovered row. The cap is inert whenever `expectedPublicQuadsDigest` is + * known, because candidates are then already content-identical and the + * first match wins; it only bites when the evidence carried no digest + * (`VerifiedGraphScopedFinalizationEvidence.publicQuadsDigest` is + * optional), which is exactly the case that could otherwise do 16 full + * payload verifications on one receipt. Truncation is logged, never + * silent. */ private async verifyImmutableGraphScopedSnapshot(input: { contextGraphId: string; @@ -1416,7 +1447,15 @@ export class FinalizationHandler { } } - for (const shareOperationId of [...new Set(shareOperationIds)]) { + const candidates = [...new Set(shareOperationIds)]; + if (candidates.length > MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS) { + this.log.warn( + input.ctx, + `Finalization: ${candidates.length} immutable snapshot candidates for ` + + `${input.scope.ual}; verifying the first ${MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS}`, + ); + } + for (const shareOperationId of candidates.slice(0, MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS)) { let quads: Quad[]; try { const snapshot = await resolveKnowledgeAssetOperationPublicQuads({ @@ -1428,6 +1467,7 @@ export class FinalizationHandler { assertionVersion: input.scope.assertionVersion, subGraphName: input.subGraphName, publicSnapshotStore: this.publicSnapshotStore, + queryOptions: { source: 'agent.finalization.resolveImmutableSnapshotPayload' }, }); quads = snapshot.quads.map((quad) => ({ ...quad, graph: '' })); } catch (error) { diff --git a/packages/publisher/src/workspace-resolution.ts b/packages/publisher/src/workspace-resolution.ts index 1f992be921..694bc656e8 100644 --- a/packages/publisher/src/workspace-resolution.ts +++ b/packages/publisher/src/workspace-resolution.ts @@ -574,6 +574,12 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { assertionVersion: string | number | bigint; subGraphName?: string; publicSnapshotStore?: WorkspacePublicSnapshotStore; + /** + * Scheduler attribution for the reads below. Optional for callers that run + * off a request they already account for; supply it from lanes where this + * resolution is itself the cost being measured (finalization receipts). + */ + queryOptions?: QueryOptions; }): Promise { const expectedScope = createGraphKnowledgeAssetScope( params.kaUal, @@ -598,6 +604,7 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { OPTIONAL { <${assertSafeIri(subject)}> <${DKG}publisherPeerId> ?publisherPeerId } } } LIMIT 1`, + params.queryOptions, ); if (result.type !== 'bindings') { throw new Error( @@ -609,6 +616,7 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { const existence = await params.store.query( `ASK { GRAPH <${assertSafeIri(workspaceMetaGraph)}> { ` + `<${assertSafeIri(subject)}> ?predicate ?object } }`, + params.queryOptions, ); if (existence.type !== 'boolean') { throw new Error( @@ -663,7 +671,7 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { `share operation ${params.shareOperationId}: unsafe snapshot graph`, ); } - quads = await resolveSnapshotGraphQuads(params.store, snapshotGraph); + quads = await resolveSnapshotGraphQuads(params.store, snapshotGraph, params.queryOptions); } if (!quads) { throw new KnowledgeAssetOperationPublicSnapshotNotFoundError( @@ -1208,9 +1216,14 @@ function workspaceKnowledgeAssetOperationSnapshotGraph( return graph; } -async function resolveSnapshotGraphQuads(store: TripleStore, snapshotGraph: string): Promise { +async function resolveSnapshotGraphQuads( + store: TripleStore, + snapshotGraph: string, + queryOptions?: QueryOptions, +): Promise { const result = await store.query( `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(snapshotGraph)}> { ?s ?p ?o } }`, + queryOptions, ); return result.type === 'quads' ? result.quads.map((quad) => ({ ...quad, graph: '' })) From 88b56ef17d9ce650075a4bc1eb4095d8b5227322 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:05:31 +0200 Subject: [PATCH 19/48] fix(agent): resume finalized-SWM sweeps from a rotation cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runSweep() always restarted at contextGraphIds[0]; the only persisted state was the backlog metrics. On budget exhaustion the worker re-woke after retryDelayMs into the same prefix, so markers in later context graphs were never discovered and every retry interval burned a full enumeration for zero progress. Persist the unvisited tail of the rotation instead. Entry 0 doubles as the resumption cursor, and the cursor advances only at a fully-measured context-graph boundary, so a sweep that yields part-way through one re-measures it whole rather than double-counting it. Because the start rotates, backlogDepth would otherwise silently become a partial sum. Accumulate per-context-graph contributions across the rotation and publish only when it closes, so backlogDepth keeps meaning "whole-node total" — it can be stale, never partial. The new `stale` flag carries that, and also fixes the counters reading 0/null under sustained load: backlogDepth/oldestMarkerAt were assigned only after a complete pressure-free sweep, so the SLO surface showed depth 0 / age null while markers accumulated and only pressureSkips moved. An operator could not tell "deferred under load" from "nothing to do". Surfaced as backlogStale on /api/slo, initially true — never measured is not empty. It adds no scan under pressure. Also guarantee forward progress: a context graph whose enumeration cannot finish inside one slice would hold the cursor forever and starve everything behind it, which is the same failure the cursor exists to fix. After three consecutive slices fail to finish the head, it moves to the back of the rotation and is retried in place. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 213 ++++++++++++++---- .../agent/src/finalized-swm-cleanup-worker.ts | 22 +- packages/cli/src/daemon/routes/agent-chat.ts | 8 + packages/cli/test/api-slo-route.test.ts | 4 + 4 files changed, 199 insertions(+), 48 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index f45c0ba8ed..bb0c02d1dd 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -66,8 +66,16 @@ export interface FinalizedSwmCleanupServiceOptions { store: TripleStore; writeLocks?: Map>; eventBus?: EventBus; - listContextGraphIds: () => Promise; - listSharedMemoryMetaGraphs: (contextGraphId: string) => Promise; + /** + * Both enumerations are handed the sweep's background query options, so an + * implementation that reaches the store keeps discovery off the foreground + * lane: a cold or past-revalidation graph-set index resolves its refresh + * priority from the caller (see GraphSetIndexStore.refreshPriority), where an + * absent priority defaults to `normal`. An implementation backed by a shared + * cache may ignore them — see the production wiring for why. + */ + listContextGraphIds: (options: QueryOptions) => Promise; + listSharedMemoryMetaGraphs: (contextGraphId: string, options: QueryOptions) => Promise; now?: () => number; maxCandidatesPerSweep?: number; wallClockBudgetMs?: number; @@ -82,15 +90,38 @@ export class FinalizedSwmCleanupService { private readonly store: TripleStore; private readonly writeLocks: Map> | undefined; private readonly eventBus: EventBus | undefined; - private readonly listContextGraphIds: () => Promise; - private readonly listSharedMemoryMetaGraphs: (contextGraphId: string) => Promise; + private readonly listContextGraphIds: (options: QueryOptions) => Promise; + private readonly listSharedMemoryMetaGraphs: ( + contextGraphId: string, + options: QueryOptions, + ) => Promise; private readonly now: () => number; private readonly maxCandidatesPerSweep: number; private readonly wallClockBudgetMs: number; private readonly graphWriteGen: GraphWriteGenSource | null; private readonly log = new Logger('FinalizedSwmCleanupService'); + /** Whole-node totals from the last rotation that closed; never a partial sum. */ private lastKnownBacklogDepth = 0; private lastKnownOldestMarkerAt: number | null = null; + /** + * Context graphs still owed a visit in the rotation currently in progress, or + * `null` when none is. Entry 0 doubles as the resumption cursor: a sweep that + * yields on pressure or wall clock persists its remaining tail here, so the + * next sweep continues instead of re-walking the same prefix forever and + * never reaching markers in later context graphs. + */ + private rotationPending: string[] | null = null; + private rotationBacklogDepth = 0; + private rotationOldestMarkerAt: number | null = null; + /** + * Head of the rotation the previous sweep started from, and how many + * consecutive sweeps have started from it without completing it. A context + * graph whose enumeration cannot finish inside one slice would otherwise hold + * the cursor forever and starve every context graph behind it — the same + * "later context graphs are never reached" failure the cursor exists to fix. + */ + private rotationHeadContextGraphId: string | null = null; + private rotationHeadStalledSweeps = 0; constructor(options: FinalizedSwmCleanupServiceOptions) { this.store = options.store; @@ -109,48 +140,71 @@ export class FinalizedSwmCleanupService { /** Run one bounded, idle-only maintenance slice. */ async runSweep(): Promise { - const pressureResult = (deletedItems = 0): FinalizedSwmCleanupSweepResult => ({ - backlogDepth: this.lastKnownBacklogDepth, - oldestMarkerAt: this.lastKnownOldestMarkerAt, - deletedItems, - pressureSkipped: true, - }); - const budgetResult = ( - backlogDepth: number, - oldestMarkerAt: number | null, + // A deferred sweep republishes the last rotation's whole-node totals rather + // than a partial sum, and marks them stale so the SLO surface can tell + // "deferred under load" apart from "nothing to do" — the distinction an + // operator needs while only `pressureSkips` is moving. + const deferredResult = ( deletedItems: number, + reason: 'pressure' | 'budget', ): FinalizedSwmCleanupSweepResult => ({ - backlogDepth: Math.max(backlogDepth, this.lastKnownBacklogDepth), - oldestMarkerAt: oldestMarkerAt ?? this.lastKnownOldestMarkerAt, + backlogDepth: this.lastKnownBacklogDepth, + oldestMarkerAt: this.lastKnownOldestMarkerAt, deletedItems, - pressureSkipped: false, - budgetExhausted: true, + pressureSkipped: reason === 'pressure', + ...(reason === 'budget' ? { budgetExhausted: true } : {}), + stale: true, }); const underPressure = () => hasActiveStorePressure(this.store.getPressureSnapshot?.()); - if (underPressure()) return pressureResult(); + if (underPressure()) return deferredResult(0, 'pressure'); const deadline = this.now() + this.wallClockBudgetMs; const deadlineSignal = AbortSignal.timeout(this.wallClockBudgetMs); + const discoveryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup.discover', + signal: deadlineSignal, + }; let remaining = this.maxCandidatesPerSweep; let deletedItems = 0; - let backlogDepth = 0; - let oldestMarkerAt: number | null = null; // Pressure is checked before every discovery boundary, including the first // potentially expensive context-graph enumeration. - if (underPressure()) return pressureResult(); - const contextGraphIds = await this.listContextGraphIds(); - for (const contextGraphId of contextGraphIds) { - if (underPressure()) return pressureResult(deletedItems); - if (this.now() >= deadline) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); + if (underPressure()) return deferredResult(0, 'pressure'); + let contextGraphIds: string[]; + try { + contextGraphIds = await this.listContextGraphIds(discoveryOptions); + } catch (error) { + if (deadlineSignal.aborted) return deferredResult(0, 'budget'); + throw error; + } + const pending = this.resumeRotation(contextGraphIds); + + for (let index = 0; index < pending.length; index += 1) { + const contextGraphId = pending[index]!; + // Yielding keeps the unvisited tail, so the cursor never advances past a + // context graph this sweep did not finish measuring. + const yieldRotation = (reason: 'pressure' | 'budget'): FinalizedSwmCleanupSweepResult => { + this.rotationPending = pending.slice(index); + return deferredResult(deletedItems, reason); + }; + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + let metaGraphs: string[]; + try { + metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId, discoveryOptions); + } catch (error) { + if (deadlineSignal.aborted) return yieldRotation('budget'); + throw error; } - const metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId); + // A context graph contributes to the rotation total only once every one of + // its meta graphs has been measured. Yielding part-way discards the partial + // sum, so the re-measure on resume cannot double-count it. + let contextGraphBacklogDepth = 0; + let contextGraphOldestMarkerAt: number | null = null; for (const swmMetaGraph of metaGraphs) { - if (underPressure()) return pressureResult(deletedItems); - if (this.now() >= deadline) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); if (remaining > 0) { let cleanup: { deletedItems: number; examinedCandidates: number }; try { @@ -161,41 +215,106 @@ export class FinalizedSwmCleanupService { signal: deadlineSignal, }); } catch (error) { - if (deadlineSignal.aborted) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } + if (deadlineSignal.aborted) return yieldRotation('budget'); throw error; } deletedItems += cleanup.deletedItems; remaining -= cleanup.examinedCandidates; } - if (underPressure()) return pressureResult(deletedItems); - if (this.now() >= deadline) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + // Deliberately not gated on `remaining`: that budget bounds deletion + // work (head resolution, VM/SWM verification, lock, atomic replace), + // while this is one cheap background aggregate and the sole source of + // the backlog metric. Capping the metric with the deletion budget would + // make it under-report exactly when the backlog is deepest. let backlog: { depth: number; oldestMarkerAt: number | null }; try { backlog = await this.inspectBacklog(swmMetaGraph, deadlineSignal); } catch (error) { - if (deadlineSignal.aborted) { - return budgetResult(backlogDepth, oldestMarkerAt, deletedItems); - } + if (deadlineSignal.aborted) return yieldRotation('budget'); throw error; } - backlogDepth += backlog.depth; + contextGraphBacklogDepth += backlog.depth; if ( backlog.oldestMarkerAt !== null - && (oldestMarkerAt === null || backlog.oldestMarkerAt < oldestMarkerAt) + && ( + contextGraphOldestMarkerAt === null + || backlog.oldestMarkerAt < contextGraphOldestMarkerAt + ) ) { - oldestMarkerAt = backlog.oldestMarkerAt; + contextGraphOldestMarkerAt = backlog.oldestMarkerAt; } await new Promise((resolve) => setImmediate(resolve)); } + this.rotationBacklogDepth += contextGraphBacklogDepth; + if ( + contextGraphOldestMarkerAt !== null + && ( + this.rotationOldestMarkerAt === null + || contextGraphOldestMarkerAt < this.rotationOldestMarkerAt + ) + ) { + this.rotationOldestMarkerAt = contextGraphOldestMarkerAt; + } + } + + // The rotation closed: every context graph was measured exactly once, so the + // accumulated sums are a whole-node total again. + this.lastKnownBacklogDepth = this.rotationBacklogDepth; + this.lastKnownOldestMarkerAt = this.rotationOldestMarkerAt; + this.rotationPending = null; + this.rotationBacklogDepth = 0; + this.rotationOldestMarkerAt = null; + this.rotationHeadContextGraphId = null; + this.rotationHeadStalledSweeps = 0; + return { + backlogDepth: this.lastKnownBacklogDepth, + oldestMarkerAt: this.lastKnownOldestMarkerAt, + deletedItems, + pressureSkipped: false, + stale: false, + }; + } + + /** + * Continue the rotation in progress, dropping context graphs that disappeared + * since it started, or open a fresh one. Context graphs created mid-rotation + * join the next rotation rather than this one; `stale` reports the lag. + * + * Guarantees forward progress: a head that three consecutive sweeps could not + * finish moves to the back of the rotation so the context graphs behind it are + * still served. It keeps its place in the rotation and is retried there. + */ + private resumeRotation(contextGraphIds: string[]): string[] { + let pending: string[]; + if (this.rotationPending === null) { + this.rotationBacklogDepth = 0; + this.rotationOldestMarkerAt = null; + pending = [...contextGraphIds]; + } else { + const known = new Set(contextGraphIds); + pending = this.rotationPending.filter((contextGraphId) => known.has(contextGraphId)); } - this.lastKnownBacklogDepth = backlogDepth; - this.lastKnownOldestMarkerAt = oldestMarkerAt; - return { backlogDepth, oldestMarkerAt, deletedItems, pressureSkipped: false }; + const head = pending[0] ?? null; + if (head !== null && head === this.rotationHeadContextGraphId) { + this.rotationHeadStalledSweeps += 1; + } else { + this.rotationHeadContextGraphId = head; + this.rotationHeadStalledSweeps = 0; + } + if (this.rotationHeadStalledSweeps >= 2 && pending.length > 1) { + this.log.warn( + createOperationContext('system'), + `Deferring context graph ${head} in finalized-SWM cleanup rotation; ` + + `${this.rotationHeadStalledSweeps + 1} consecutive slices could not finish it`, + ); + pending = [...pending.slice(1), pending[0]!]; + this.rotationHeadContextGraphId = pending[0] ?? null; + this.rotationHeadStalledSweeps = 0; + } + return pending; } /** Narrow test seam for one already-known SWM metadata graph. */ diff --git a/packages/agent/src/finalized-swm-cleanup-worker.ts b/packages/agent/src/finalized-swm-cleanup-worker.ts index 04f4108da7..3671094714 100644 --- a/packages/agent/src/finalized-swm-cleanup-worker.ts +++ b/packages/agent/src/finalized-swm-cleanup-worker.ts @@ -10,7 +10,12 @@ */ export interface FinalizedSwmCleanupSweepResult { - /** Cleanup tasks still requiring a safe idle pass after this sweep. */ + /** + * Cleanup tasks still requiring a safe idle pass, as a whole-node total from + * the last context-graph rotation that ran to completion. Never a partial sum: + * a sweep that only covers part of a rotation republishes the previous total + * and sets {@link stale}. + */ backlogDepth: number; /** Oldest surviving cleanup marker, or null when the backlog is empty/unknown. */ oldestMarkerAt: number | null; @@ -20,11 +25,23 @@ export interface FinalizedSwmCleanupSweepResult { pressureSkipped: boolean; /** True when the bounded slice yielded with more discovery work possible. */ budgetExhausted?: boolean; + /** + * True when `backlogDepth`/`oldestMarkerAt` are not a current measurement — + * this sweep deferred on pressure or wall clock, or is part-way through a + * rotation. Distinguishes a deferred backlog from an empty one. + */ + stale?: boolean; } export interface FinalizedSwmCleanupStats { backlogDepth: number; oldestMarkerAgeMs: number | null; + /** + * True until a full context-graph rotation has published a whole-node + * measurement, and again whenever the latest sweep deferred. While true, + * `backlogDepth` 0 / `oldestMarkerAgeMs` null mean "unknown", not "empty". + */ + backlogStale: boolean; pressureSkips: number; deletedItems: number; runs: number; @@ -56,6 +73,8 @@ export class FinalizedSwmCleanupWorker { private readonly stats: FinalizedSwmCleanupStats = { backlogDepth: 0, oldestMarkerAgeMs: null, + // Nothing has been measured yet, which is not the same as an empty backlog. + backlogStale: true, pressureSkips: 0, deletedItems: 0, runs: 0, @@ -148,6 +167,7 @@ export class FinalizedSwmCleanupWorker { try { const result = await this.sweep(); this.stats.backlogDepth = Math.max(0, Math.floor(result.backlogDepth)); + this.stats.backlogStale = result.stale === true; this.oldestMarkerAt = result.oldestMarkerAt; this.stats.oldestMarkerAgeMs = this.oldestMarkerAt === null ? null diff --git a/packages/cli/src/daemon/routes/agent-chat.ts b/packages/cli/src/daemon/routes/agent-chat.ts index 4378356783..87ed84f4fe 100644 --- a/packages/cli/src/daemon/routes/agent-chat.ts +++ b/packages/cli/src/daemon/routes/agent-chat.ts @@ -1177,6 +1177,7 @@ export function buildSloPayload(agent: { getFinalizedSwmCleanupStats?: () => { backlogDepth: number; oldestMarkerAgeMs: number | null; + backlogStale: boolean; pressureSkips: number; deletedItems: number; runs: number; @@ -1232,6 +1233,13 @@ export function buildSloPayload(agent: { finalizedCleanup?: { backlogDepth: number; oldestMarkerAgeMs: number | null; + /** + * True while `backlogDepth`/`oldestMarkerAgeMs` are not a current + * whole-node measurement — the GC deferred on store pressure or its slice + * budget, or is part-way through a context-graph rotation. Depth 0 with + * this set means "unknown", not "drained". + */ + backlogStale: boolean; pressureSkips: number; deletedItems: number; runs: number; diff --git a/packages/cli/test/api-slo-route.test.ts b/packages/cli/test/api-slo-route.test.ts index 3136a4473b..43400915d9 100644 --- a/packages/cli/test/api-slo-route.test.ts +++ b/packages/cli/test/api-slo-route.test.ts @@ -295,6 +295,7 @@ describe('/api/slo wire format (rc.9 PR-A / Codex PR #570 R10)', () => { getFinalizedSwmCleanupStats: () => ({ backlogDepth: 12, oldestMarkerAgeMs: 45_000, + backlogStale: true, pressureSkips: 7, deletedItems: 31, runs: 9, @@ -309,6 +310,9 @@ describe('/api/slo wire format (rc.9 PR-A / Codex PR #570 R10)', () => { expect((body as { swm: Record }).swm.finalizedCleanup).toEqual({ backlogDepth: 12, oldestMarkerAgeMs: 45_000, + // Deferred-vs-drained is the distinction operators page on; it must reach + // the wire, not just the in-process snapshot. + backlogStale: true, pressureSkips: 7, deletedItems: 31, runs: 9, From bf7971471b6a55e621b61f05d3f15ded5b1573fa Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:05:46 +0200 Subject: [PATCH 20/48] fix(agent): run finalized-SWM meta-graph discovery on the background lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-context-graph SWM meta-graph enumeration called listGraphsByPrefix(store, cgPrefix) bare. The store scheduler defaults an absent priority to 'normal', and GraphSetIndexStore forces a background refresh only when the index is dirty — a seed or revalidate refresh uses `options?.priority ?? 'normal'`. So on a cold or past-revalidation index the GC's own enumeration became a full foreground listGraphs() scan. Thread the sweep's background options, and its deadline signal with them, so a long enumeration is cut short instead of overrunning the slice. Both enumeration calls now yield the slice on abort rather than escaping to the worker's error path. The context-graph id listing deliberately keeps using the shared cached DKGAgent.listContextGraphs() and ignores the options: it is the only source that resolves owner/name context graphs (a graph-URI walk cannot tell `/` apart from a sub-graph), its cost is amortized across foreground callers, and threading the sweep's abort signal into a promise other callers join would let them inherit the GC's deadline. The per-context-graph call is the one that scales with graph count. Make the slice budgets tunable rather than defaults-only at the single production construction site, following the VM_RECONCILE_* convention: DKG_FINALIZED_SWM_CLEANUP_MAX_CANDIDATES, _BUDGET_MS and _RETRY_MS. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/dkg-agent-base.ts | 18 +++++++++++ packages/agent/src/dkg-agent-lifecycle.ts | 37 +++++++++++++++++------ 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index 24e66be7c6..60d5e24aa7 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -881,6 +881,24 @@ export class DKGAgentBase { */ static readonly SWM_ACK_QUORUM_TICK_MS = 5_000; + /** + * Finalized-SWM GC slice budgets. The sweep yields once either is spent and + * resumes at the next context graph in its rotation, so these trade cleanup + * latency against how long one background slice may hold store capacity. + * Env-overridable for ops tuning; the service clamps candidates to 16. + */ + static readonly FINALIZED_SWM_CLEANUP_MAX_CANDIDATES = + Math.max(1, Number(process.env['DKG_FINALIZED_SWM_CLEANUP_MAX_CANDIDATES']) || 4); + static readonly FINALIZED_SWM_CLEANUP_BUDGET_MS = + Math.max(1, Number(process.env['DKG_FINALIZED_SWM_CLEANUP_BUDGET_MS']) || 10_000); + /** + * Delay before re-waking the GC after a sweep yielded on pressure or budget, + * or drained part of a known backlog. This is the cadence that actually fires + * while the node is draining, so it is the knob ops reach for first. + */ + static readonly FINALIZED_SWM_CLEANUP_RETRY_MS = + Math.max(1, Number(process.env['DKG_FINALIZED_SWM_CLEANUP_RETRY_MS']) || 5_000); + /** * Phase B — chain-driven VM reconciliation sweep cadence. The periodic sweep * is the safety net behind the live `KnowledgeAssetRegisteredToContextGraph` diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 21c50d73eb..cce2715120 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -98,7 +98,7 @@ import { pickNetworkTunables, withRetry, } from '@origintrail-official/dkg-core'; -import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type QueryOptions, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; import { readChangelogDeltaPage } from './sync/responder/graph-plan.js'; import { decodeChangelogRequest, encodeChangelogResponse } from './sync/changelog/wire.js'; import { runChangelogSync, planPageApply } from './sync/requester/changelog-sync.js'; @@ -7614,7 +7614,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (!this.finalizedSwmCleanupWorker) { this.finalizedSwmCleanupWorker = new FinalizedSwmCleanupWorker({ sweep: () => this.runFinalizedSwmCleanupSweep(), - retryDelayMs: 5_000, + retryDelayMs: DKGAgentBase.FINALIZED_SWM_CLEANUP_RETRY_MS, onError: (error) => { this.log.warn( createOperationContext('system'), @@ -7641,9 +7641,20 @@ export class LifecycleSyncMethods extends DKGAgentBase { store: this.store, writeLocks: this.writeLocks, eventBus: this.eventBus, + // Deliberately ignores the sweep's query options: this is the shared, + // TTL-cached listing, so its result is normally amortized across + // foreground callers rather than store work attributable to the GC, and + // threading the sweep's deadline signal in would let a foreground caller + // joining the same in-flight promise inherit the GC's abort. It stays + // the id source because it is the only one that resolves owner/name + // context graphs — a graph-URI walk cannot tell `/` apart + // from a sub-graph. The per-context-graph enumeration below is the call + // that scales with graph count, and that one is lane-tagged. listContextGraphIds: async () => (await this.listContextGraphs()).map((row) => row.id), - listSharedMemoryMetaGraphs: (contextGraphId) => - listSharedMemoryMetaGraphs(this.store, contextGraphId), + listSharedMemoryMetaGraphs: (contextGraphId, options) => + listSharedMemoryMetaGraphs(this.store, contextGraphId, options), + maxCandidatesPerSweep: DKGAgentBase.FINALIZED_SWM_CLEANUP_MAX_CANDIDATES, + wallClockBudgetMs: DKGAgentBase.FINALIZED_SWM_CLEANUP_BUDGET_MS, }); } return this.finalizedSwmCleanupService; @@ -7844,10 +7855,14 @@ async function listGraphFamily(store: TripleStore, rootGraph: string): Promise { +async function listGraphsByPrefix( + store: TripleStore, + prefix: string, + options?: QueryOptions, +): Promise { return store.listGraphsByPrefix - ? store.listGraphsByPrefix(prefix) - : (await store.listGraphs()).filter((graph) => graph.startsWith(prefix)); + ? store.listGraphsByPrefix(prefix, options) + : (await store.listGraphs(options)).filter((graph) => graph.startsWith(prefix)); } async function runRecoverContextGraphSwmFromPeer( @@ -8054,12 +8069,16 @@ async function isKnownContextGraphUri(store: TripleStore, contextGraphUri: strin * families such as `…/_verifiable_memory/…` or `…/_shared_memory_snapshots/…` * can never be misread as a sub-graph meta graph. */ -async function listSharedMemoryMetaGraphs(store: TripleStore, contextGraphId: string): Promise { +async function listSharedMemoryMetaGraphs( + store: TripleStore, + contextGraphId: string, + options?: QueryOptions, +): Promise { const rootMetaGraph = contextGraphWorkspaceMetaGraphUri(contextGraphId); const cgPrefix = `did:dkg:context-graph:${contextGraphId}/`; const metaSuffix = '/_shared_memory_meta'; const metaGraphs = [rootMetaGraph]; - for (const graph of await listGraphsByPrefix(store, cgPrefix)) { + for (const graph of await listGraphsByPrefix(store, cgPrefix, options)) { if (graph === rootMetaGraph || !graph.endsWith(metaSuffix)) continue; const subGraphName = graph.slice(cgPrefix.length, graph.length - metaSuffix.length); if (!validateSubGraphName(subGraphName).valid) continue; From 492726645669bcb70087f701cb4a2e853d1b5ac0 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:14:06 +0200 Subject: [PATCH 21/48] docs(agent): scope the finalized-SWM discovery signal claim The deadline signal bounds enumeration by wall clock, which nothing did before, but it is not load-responsive preemption: pressure is sampled at boundaries only, and the background lane bounds concurrency rather than interrupting a query already running. Say so, so the gap is not read as covered. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/finalized-swm-cleanup-service.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index bb0c02d1dd..e1edc18283 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -160,6 +160,11 @@ export class FinalizedSwmCleanupService { const deadline = this.now() + this.wallClockBudgetMs; const deadlineSignal = AbortSignal.timeout(this.wallClockBudgetMs); + // The signal bounds enumeration by wall clock, which nothing did before. + // It is not load-responsive preemption: pressure is sampled at boundaries + // only, and the background lane bounds concurrency rather than interrupting + // a query already running. Cutting an in-flight scan short when load + // arrives mid-enumeration remains an accepted gap. const discoveryOptions: QueryOptions = { priority: 'background', source: 'agent.finalizedSwmCleanup.discover', From 4c9d937074b4ed368ebd499a473e67232617142a Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:20:17 +0200 Subject: [PATCH 22/48] fix(agent): correct the stalled-head deferral count and rotation docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deferral warning reported one more failed attempt than occurred. resumeRotation increments only when the head repeats, so the counter reaching 2 means two slices attempted and failed; the third defers before attempting again. Report the counter as-is. Operator-facing text that overstates its evidence is worse than no text. Also correct two comments that claimed behaviour the code does not have: a closing rotation reports stale:false even though context graphs created mid-rotation join the next one, and that is deliberate — flagging the lag would make `stale` true almost always on a node with any context-graph churn, destroying the deferred-versus-drained signal it exists to carry. Document the empty-tail close, reached when every context graph still owed a visit disappears mid-rotation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index e1edc18283..8c35e5c891 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -265,7 +265,10 @@ export class FinalizedSwmCleanupService { } // The rotation closed: every context graph was measured exactly once, so the - // accumulated sums are a whole-node total again. + // accumulated sums are a whole-node total again. Reached with an empty tail + // too, when every context graph still owed a visit disappeared mid-rotation + // (or the node has none at all) — the total then covers exactly the graphs + // that still exist, which is the whole-node answer. this.lastKnownBacklogDepth = this.rotationBacklogDepth; this.lastKnownOldestMarkerAt = this.rotationOldestMarkerAt; this.rotationPending = null; @@ -285,11 +288,16 @@ export class FinalizedSwmCleanupService { /** * Continue the rotation in progress, dropping context graphs that disappeared * since it started, or open a fresh one. Context graphs created mid-rotation - * join the next rotation rather than this one; `stale` reports the lag. + * join the next rotation rather than this one, so a closing total covers the + * set as of rotation start; the lag is bounded by one rotation. That lag is + * deliberately not reported as `stale` — on a node with any context-graph + * churn the flag would be true almost always, destroying the deferred-versus- + * drained signal it exists to carry. * - * Guarantees forward progress: a head that three consecutive sweeps could not - * finish moves to the back of the rotation so the context graphs behind it are - * still served. It keeps its place in the rotation and is retried there. + * Guarantees forward progress: after two consecutive slices fail to finish the + * head, the third defers it to the back of the rotation so the context graphs + * behind it are still served. It keeps its place in the rotation and is + * retried there. */ private resumeRotation(contextGraphIds: string[]): string[] { let pending: string[]; @@ -313,7 +321,7 @@ export class FinalizedSwmCleanupService { this.log.warn( createOperationContext('system'), `Deferring context graph ${head} in finalized-SWM cleanup rotation; ` - + `${this.rotationHeadStalledSweeps + 1} consecutive slices could not finish it`, + + `${this.rotationHeadStalledSweeps} consecutive slices could not finish it`, ); pending = [...pending.slice(1), pending[0]!]; this.rotationHeadContextGraphId = pending[0] ?? null; From 578fe95fa6de8e98933cce325e6b8d64b458b1c7 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:31:01 +0200 Subject: [PATCH 23/48] fix(agent): dedupe late-receipt snapshot work by content, not by candidate cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS cap added in 5df21041d. That cap bounded a SEARCH, not work. With `ORDER BY ?shareId` the candidate order is unrelated to which candidate matches, so truncating the list does not do less work — it returns a different answer. The reachable case is exactly the one the cap was aimed at: the immutable-snapshot path requires trustedAssertionEvidence, whose publicQuadsDigest is OPTIONAL, so with no digest the candidates are filtered by triple count alone. When the match sorts past the cap, verifyImmutableGraphScopedSnapshot returns undefined, the caller falls back to a live SWM layer the GC has already drained, and the receipt resolves to 'no-swm'. That leaves the cursor for a sweep retry, but the candidate list is deterministic, so every retry re-derives the identical truncation: the KA is stranded, not delayed. Bound the repeated work instead. resolveKnowledgeAssetOperationPublicQuads throws unless the payload hashes to the digest stored on that operation subject, workspacePublicQuadsDigest sorts its rows before hashing, and V10MerkleTree sorts and dedupes its leaves — so equal digest implies equal count and an identical Merkle root. Once a fully resolved candidate has been rejected, every other candidate advertising that digest is provably a repeat, and skipping it cannot change the answer. A throw does not memo: that means this operation's snapshot was unreadable, not that the content is wrong, and a sibling may still hold a readable copy. With a digest the memo collapses the list to one payload read; without one the candidates genuinely differ and the residue stays bounded by the existing discovery LIMIT, as it is today. Also drops FinalizationHandler.writeLocks, dead since 4b1a2a9ff removed discardFinalizedGraphAsset. It implied the finalization path held the per-KA SWM writer lock when it never did. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/dkg-agent-swm-substrate.ts | 1 - packages/agent/src/finalization-handler.ts | 84 ++++++++++--------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index d42ea81e54..90e5427dc5 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -1654,7 +1654,6 @@ export class SwmSubstrateMethods extends DKGAgentBase { markContextGraphMetaDirtyFromQuads: (quads) => { this.contextGraphMetaProjection.markDirtyFromQuads(quads); }, - writeLocks: this.writeLocks, publicSnapshotStore: this.publicSnapshotStore, wakeFinalizedSwmCleanup: () => this.wakeFinalizedSwmCleanup(), runtime: this.finalizationRuntime, diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index b20a9fa3e7..ae6806f6f5 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -211,18 +211,6 @@ type GraphScopedMaterializationEnvelope = Pick< /** Immutable queued assertion envelope supplied only after receipt/seal validation. */ type TrustedGraphScopedAssertionEvidence = VerifiedGraphScopedFinalizationEvidence; -/** - * How many immutable operation snapshots one receipt may verify in full. - * - * Discovery is allowed to surface more candidates than this — it is a cheap - * bounded metadata read — but each verification costs a whole payload read - * plus digest plus Merkle root, on the latency-sensitive receipt path. Bound - * the expensive half so that a KA re-shared many times cannot turn one late - * receipt into a multiple of that work. See - * `FinalizationHandler.verifyImmutableGraphScopedSnapshot`. - */ -const MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS = 4; - function resolveGraphScopedAccessEnvelope( head: GraphScopedMaterializationEnvelope, requestedAccessPolicy?: GraphScopedAccessPolicy, @@ -304,7 +292,6 @@ export interface FinalizationHandlerOptions { eventBus?: EventBus; resolveContextGraphOnChainId?: ResolveContextGraphOnChainId; markContextGraphMetaDirtyFromQuads?: MarkContextGraphMetaDirtyFromQuads; - writeLocks?: Map>; publicSnapshotStore?: WorkspacePublicSnapshotStore; wakeFinalizedSwmCleanup?: () => void; lifecycleLogOptions?: FinalizationLifecycleLogOptions; @@ -376,7 +363,6 @@ export class FinalizationHandler { private readonly eventBus: EventBus | undefined; private readonly resolveContextGraphOnChainId: ResolveContextGraphOnChainId | undefined; private readonly markContextGraphMetaDirtyFromQuads: MarkContextGraphMetaDirtyFromQuads | undefined; - private readonly writeLocks: Map> | undefined; private readonly publicSnapshotStore: WorkspacePublicSnapshotStore | undefined; private readonly wakeFinalizedSwmCleanup: (() => void) | undefined; private readonly recovery: FinalizationRecovery; @@ -438,7 +424,6 @@ export class FinalizationHandler { this.eventBus = options.eventBus; this.resolveContextGraphOnChainId = options.resolveContextGraphOnChainId; this.markContextGraphMetaDirtyFromQuads = options.markContextGraphMetaDirtyFromQuads; - this.writeLocks = options.writeLocks; this.publicSnapshotStore = options.publicSnapshotStore; this.wakeFinalizedSwmCleanup = options.wakeFinalizedSwmCleanup; this.lifecycle = new FinalizationLifecycleLogger( @@ -1376,23 +1361,23 @@ export class FinalizationHandler { * recovery independent from retaining a second live copy of the payload. * * Once the idle GC has done its job this is the NORMAL path for a late - * receipt, not an exception, so its cost has to be bounded rather than - * merely correct. Two rules hold it down: + * receipt, not an exception, so its cost has to be held down — but this is a + * SEARCH, and the only sound way to make a search cheaper is to remove work + * that provably cannot change its answer: * * - Discovery stays a bounded metadata read (one bound `kaUal`, LIMIT 16) * and keeps the receipt lane's default priority. Receipts are * latency-sensitive; demoting them to the background lane would trade a * CPU spike for receipt starvation under load, which is the worse failure. * - Payload verification is the expensive half — a full snapshot read plus - * digest plus Merkle root PER CANDIDATE — so it is capped at - * {@link MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS} instead of running once per - * discovered row. The cap is inert whenever `expectedPublicQuadsDigest` is - * known, because candidates are then already content-identical and the - * first match wins; it only bites when the evidence carried no digest - * (`VerifiedGraphScopedFinalizationEvidence.publicQuadsDigest` is - * optional), which is exactly the case that could otherwise do 16 full - * payload verifications on one receipt. Truncation is logged, never - * silent. + * digest plus Merkle root per candidate — and it is deduplicated BY + * CONTENT (see the memo below), not truncated by candidate count. When the + * evidence carries a digest, every candidate advertises that same digest, + * so the memo collapses the whole list to one payload read. When it does + * not (`VerifiedGraphScopedFinalizationEvidence.publicQuadsDigest` is + * optional), the candidates genuinely differ and each one has to be + * checked; that residue is inherent to the search and stays bounded by the + * discovery LIMIT. */ private async verifyImmutableGraphScopedSnapshot(input: { contextGraphId: string; @@ -1406,9 +1391,9 @@ export class FinalizationHandler { ctx: OperationContext; }): Promise | undefined> { const graphManager = new GraphManager(this.store); - const shareOperationIds: string[] = []; + const shareOperationIds: Array<{ shareOperationId: string; digest?: string }> = []; if (input.expectedHead) { - shareOperationIds.push(input.expectedHead.shareOperationId); + shareOperationIds.push({ shareOperationId: input.expectedHead.shareOperationId }); } else { const metaGraph = graphManager.sharedMemoryMetaUri( input.contextGraphId, @@ -1441,21 +1426,38 @@ export class FinalizationHandler { || digest === input.expectedPublicQuadsDigest ) ) { - shareOperationIds.push(shareId); + shareOperationIds.push({ shareOperationId: shareId, ...(digest ? { digest } : {}) }); } } } } - const candidates = [...new Set(shareOperationIds)]; - if (candidates.length > MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS) { - this.log.warn( - input.ctx, - `Finalization: ${candidates.length} immutable snapshot candidates for ` - + `${input.scope.ual}; verifying the first ${MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS}`, - ); - } - for (const shareOperationId of candidates.slice(0, MAX_IMMUTABLE_SNAPSHOT_VERIFICATIONS)) { + const seenShareOperationIds = new Set(); + const candidates = shareOperationIds.filter((candidate) => { + if (seenShareOperationIds.has(candidate.shareOperationId)) return false; + seenShareOperationIds.add(candidate.shareOperationId); + return true; + }); + // Content-level memo. `resolveKnowledgeAssetOperationPublicQuads` throws + // unless the resolved payload hashes to the digest stored on that operation + // subject, so two candidates advertising the same digest resolve to the + // same quads and therefore to the same count and Merkle root. Once one of + // them has been fully resolved and rejected, every other candidate with + // that digest is provably a repeat of work already done — skipping it + // cannot change the answer. A THROW must not memo: that means we could not + // read THAT operation's snapshot, not that the content is wrong, and a + // sibling operation may still hold a readable copy. + // + // This bounds the repeated work without bounding the SEARCH. A cap on the + // candidate count would do the opposite: with `ORDER BY ?shareId` the order + // is unrelated to which candidate matches, so truncating it does not do + // less work, it returns a different answer — `undefined`, which decays into + // 'no-swm'. That outcome leaves the cursor for a sweep retry, but the + // candidate list is deterministic, so every retry re-derives the identical + // truncation and the receipt is stranded rather than delayed. + const rejectedDigests = new Set(); + for (const { shareOperationId, digest: candidateDigest } of candidates) { + if (candidateDigest !== undefined && rejectedDigests.has(candidateDigest)) continue; let quads: Quad[]; try { const snapshot = await resolveKnowledgeAssetOperationPublicQuads({ @@ -1481,13 +1483,17 @@ export class FinalizationHandler { && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest ) ) { + if (candidateDigest !== undefined) rejectedDigests.add(candidateDigest); continue; } const merkleRoot = computeFlatKCRoot( quads, input.privateMerkleRoot ? [input.privateMerkleRoot] : [], ); - if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) continue; + if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { + if (candidateDigest !== undefined) rejectedDigests.add(candidateDigest); + continue; + } return { status: 'verified', graphUri: knowledgeAssetLayerGraphUri( From b091737aa704b408508e93bfddfbad9579f17493 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:34:23 +0200 Subject: [PATCH 24/48] test(agent): pin the finalized-SWM GC rotation cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rotation cursor shipped with no standing test: its author verified it with a throwaway harness that was deleted. On a PR whose whole subject is that the original guards were unpinned, shipping the fix unpinned repeats the mistake being corrected. The discriminating observable is which context graph the next slice enumerates, not the returned result shape — assertions on the result alone pass with the cursor deleted, because a restarting sweep produces the same shape. Each test therefore records the context graphs entered from inside the injected enumeration closure. The accumulator is pinned by the case that actually separates the two placements: a slice that yields part-way through a context graph. Twenty markers across two meta graphs of one context graph publish 20, not 30 — committing each meta graph straight into the rotation accumulator instead of at the context-graph boundary double-counts the re-measure on resume. The four-context-graph rotation does not discriminate that, because a graph entered but not finished yields before it measures anything. Also pins the starvation guard at its real threshold (two consecutive failed slices, with the head retried in place rather than dropped), the stale flag on every deferred slice, a context graph disappearing mid-rotation, and the background lane and slice deadline on the per-context-graph enumeration. Adds a deliberate owner/name guard. That property survived two independent near-misses in one session — an enumeration source that drops ids containing a slash, and a listing mode that drops context graphs with no explicit privacy policy — and was protected only incidentally. Requires the rotation-cursor commits; these tests target the integrated state, not fix-1996-qa alone. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../finalized-swm-cleanup-rotation.test.ts | 374 ++++++++++++++++++ packages/agent/vitest.unit.config.ts | 1 + 2 files changed, 375 insertions(+) create mode 100644 packages/agent/test/finalized-swm-cleanup-rotation.test.ts diff --git a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts new file mode 100644 index 0000000000..54c05974f1 --- /dev/null +++ b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Rotation-cursor behaviour of the idle finalized-SWM GC. + * + * A sweep that yields mid-rotation persists its unvisited tail, so the next + * slice continues instead of re-walking the same prefix forever. That makes + * `backlogDepth` the delicate part: with a rotating start, a naive per-sweep + * sum would be a partial number whose meaning changes every sweep. The service + * instead accumulates per-context-graph contributions and publishes only when + * the rotation closes, so the metric can be STALE but never PARTIAL. + * + * The assertions here are therefore about the published totals and about which + * context graphs each slice actually entered. Asserting only that the cursor + * advanced is much weaker: it passes with the accumulator committed in the + * wrong place, which is the mutation that silently inflates the SLO surface. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { + GraphManager, + OxigraphStore, + type Quad, + type QueryOptions, +} from '@origintrail-official/dkg-storage'; +import { + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from '../src/dkg-agent-constants.js'; +import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; + +const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; +const XSD_DATE_TIME = 'http://www.w3.org/2001/XMLSchema#dateTime'; +const MARKER_ROOT = JSON.stringify(`0x${'11'.repeat(32)}`); + +/** + * Backlog markers only. The discovery query additionally requires kaUal, + * assertionVersion, shareOperationId, assertionGraph and headFingerprint, so + * these rows are counted by `inspectBacklog` and ignored by cleanup — which is + * what keeps these tests about rotation rather than about deletion. + */ +function markerQuads(label: string, metaGraph: string, count: number): Quad[] { + const quads: Quad[] = []; + for (let index = 0; index < count; index += 1) { + const subject = `urn:dkg:finalized-swm-cleanup:${label}-${index}`; + quads.push( + { subject, predicate: RDF_TYPE, object: FINALIZED_SWM_CLEANUP_TASK_TYPE, graph: metaGraph }, + { subject, predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, object: MARKER_ROOT, graph: metaGraph }, + { + subject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: `"${new Date(1_700_000_000_000 + index * 1_000).toISOString()}"^^<${XSD_DATE_TIME}>`, + graph: metaGraph, + }, + ); + } + return quads; +} + +function installPressureSwitch(store: OxigraphStore): { busy: boolean } { + const state = { busy: false }; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: state.busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + return state; +} + +describe('finalized SWM cleanup rotation cursor', () => { + /** + * The property the whole accumulate-across-rotation design exists to hold: a + * slice that yields part-way through a context graph must discard that + * context graph's partial contribution, so the re-measure on resume cannot + * double-count it. + * + * 20 markers across two meta graphs of one context graph. The first slice + * measures one meta graph and then yields, the second measures both. The + * published total must be 20 — committing each meta graph straight into the + * rotation accumulator instead of at the context-graph boundary yields 30. + */ + it('discards a partial context-graph measurement rather than double-counting it on resume', async () => { + const store = new OxigraphStore(); + const pressure = installPressureSwitch(store); + const graphManager = new GraphManager(store); + const rootMeta = graphManager.sharedMemoryMetaUri('cg-partial'); + const subMeta = graphManager.sharedMemoryMetaUri('cg-partial', 'named'); + await store.insert(markerQuads('root', rootMeta, 10)); + await store.insert(markerQuads('sub', subMeta, 10)); + + // Pressure appears once the first meta graph has been measured, so the + // slice yields at the top of the second meta-graph iteration. + let backlogReads = 0; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + const result = await originalQuery(query, options); + if (options?.source === 'agent.finalizedSwmCleanup.backlog') { + backlogReads += 1; + if (backlogReads === 1) pressure.busy = true; + } + return result; + }); + + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => ['cg-partial'], + listSharedMemoryMetaGraphs: async () => [rootMeta, subMeta], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + stale: true, + // Nothing has ever been published, and the 10 markers measured in this + // slice are NOT it — that would be a partial sum. + backlogDepth: 0, + }); + expect(backlogReads).toBe(1); + + pressure.busy = false; + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + stale: false, + backlogDepth: 20, + }); + // Three reads total: one discarded, then both meta graphs re-measured. + expect(backlogReads).toBe(3); + querySpy.mockRestore(); + }); + + /** + * A yielding sweep must resume at its unvisited tail. Before the cursor, + * every retry re-walked the same prefix, so markers in later context graphs + * were never discovered no matter how many times the worker woke. + * + * Four context graphs, 10 markers each, enumeration costing 60ms against a + * 100ms slice. Context graphs legitimately repeat across slices — a graph + * entered but not finished is re-measured whole — so the assertion is on the + * entry sequence and on the single published total, not on disjoint sets. + */ + it('resumes at the unvisited tail and publishes a whole-node total only when the rotation closes', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const contextGraphIds = ['cg-a', 'cg-b', 'cg-c', 'cg-d']; + const metaByContextGraph = new Map(); + for (const contextGraphId of contextGraphIds) { + const meta = graphManager.sharedMemoryMetaUri(contextGraphId); + metaByContextGraph.set(contextGraphId, meta); + await store.insert(markerQuads(contextGraphId, meta, 10)); + } + + const clock = { now: 1_000_000 }; + const entered: string[][] = []; + let currentSweep: string[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 100, + listContextGraphIds: async () => [...contextGraphIds], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + currentSweep.push(contextGraphId); + clock.now += 60; + return [metaByContextGraph.get(contextGraphId)!]; + }, + }); + + const runSlice = async () => { + currentSweep = []; + const result = await service.runSweep(); + entered.push(currentSweep); + return result; + }; + + await expect(runSlice()).resolves.toMatchObject({ budgetExhausted: true, stale: true, backlogDepth: 0 }); + await expect(runSlice()).resolves.toMatchObject({ budgetExhausted: true, stale: true, backlogDepth: 0 }); + await expect(runSlice()).resolves.toMatchObject({ budgetExhausted: true, stale: true, backlogDepth: 0 }); + // The rotation closes here: every context graph has been measured exactly + // once, so 4 x 10 markers is a whole-node total. + const closing = await runSlice(); + expect(closing).toMatchObject({ pressureSkipped: false, stale: false, backlogDepth: 40 }); + expect(closing.budgetExhausted).toBeUndefined(); + + // Seven context-graph entries produced a total of 40, not 70: the three + // graphs entered twice contributed once each. + expect(entered).toEqual([ + ['cg-a', 'cg-b'], + ['cg-b', 'cg-c'], + ['cg-c', 'cg-d'], + ['cg-d'], + ]); + expect(entered.flat()).toHaveLength(7); + + // A fresh rotation opens at the head again, republishing the last complete + // total as stale rather than resetting it to zero. + await expect(runSlice()).resolves.toMatchObject({ stale: true, backlogDepth: 40 }); + expect(entered[4]).toEqual(['cg-a', 'cg-b']); + }); + + /** + * Forward progress. A context graph whose enumeration cannot finish inside + * one slice would otherwise hold the cursor forever and starve every context + * graph behind it — the same "later graphs are never reached" failure the + * cursor exists to fix. + */ + it('rotates a head that repeatedly cannot be finished so the tail is still served', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const slowMeta = graphManager.sharedMemoryMetaUri('cg-slow'); + const metaB = graphManager.sharedMemoryMetaUri('cg-b'); + const metaC = graphManager.sharedMemoryMetaUri('cg-c'); + await store.insert(markerQuads('slow', slowMeta, 5)); + await store.insert(markerQuads('b', metaB, 5)); + await store.insert(markerQuads('c', metaC, 5)); + + const clock = { now: 1_000_000 }; + const entered: string[][] = []; + let currentSweep: string[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 100, + listContextGraphIds: async () => ['cg-slow', 'cg-b', 'cg-c'], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + currentSweep.push(contextGraphId); + // The head alone costs more than a whole slice, so it can never finish. + clock.now += contextGraphId === 'cg-slow' ? 200 : 10; + if (contextGraphId === 'cg-slow') return [slowMeta]; + return [contextGraphId === 'cg-b' ? metaB : metaC]; + }, + }); + + for (let slice = 0; slice < 3; slice += 1) { + currentSweep = []; + await service.runSweep(); + entered.push(currentSweep); + } + + expect(entered[0]).toEqual(['cg-slow']); + expect(entered[1]).toEqual(['cg-slow']); + // The head steps aside after TWO failed slices and keeps its place in the + // rotation, so the graphs behind it are finally reached — and it is still + // retried in the same pass rather than dropped. + expect(entered[2]).toEqual(['cg-b', 'cg-c', 'cg-slow']); + }); + + /** + * Owner/name context graphs (`/`) must stay reachable by GC + * discovery. Nothing about that is obvious from the code, and it has already + * survived two independent near-misses: an enumeration source that drops ids + * containing `/`, and a listing mode that drops context graphs with no + * explicit privacy policy. Either would have made these graphs invisible to + * the GC and left their finalized SWM copies forever — #1996 reintroduced by + * the back door, in the subsystem meant to close it. + * + * This pins the service side: an id containing `/` must survive graph-URI + * derivation, discovery and backlog measurement. Which enumeration source + * feeds `listContextGraphIds` is production wiring and is covered separately + * by the agent-level TTL suite. + */ + it('discovers and measures an owner/name context graph', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const ownerNameId = `${'0x1111111111111111111111111111111111111111'}/public-finalized-cleanup`; + const rootMeta = graphManager.sharedMemoryMetaUri(ownerNameId); + const subMeta = graphManager.sharedMemoryMetaUri(ownerNameId, 'named-slice'); + expect(rootMeta).toContain(ownerNameId); + await store.insert(markerQuads('owner-root', rootMeta, 4)); + await store.insert(markerQuads('owner-sub', subMeta, 2)); + + const enumerated: string[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [ownerNameId], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + enumerated.push(contextGraphId); + return [rootMeta, subMeta]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + stale: false, + // Both the root and the named sub-graph meta graph of the owner/name CG + // were measured; a `/`-rejecting path anywhere here reports 0. + backlogDepth: 6, + }); + expect(enumerated).toEqual([ownerNameId]); + }); + + /** + * A context graph deleted mid-rotation must not strand the rotation. The + * pending tail is re-filtered against a fresh enumeration each slice. + */ + it('drops a context graph that disappears mid-rotation without stranding the rotation', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const metaA = graphManager.sharedMemoryMetaUri('cg-a'); + const metaB = graphManager.sharedMemoryMetaUri('cg-b'); + await store.insert(markerQuads('a', metaA, 7)); + await store.insert(markerQuads('b', metaB, 3)); + + const clock = { now: 1_000_000 }; + let live = ['cg-a', 'cg-b']; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 100, + listContextGraphIds: async () => [...live], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + clock.now += 60; + return [contextGraphId === 'cg-a' ? metaA : metaB]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ stale: true, backlogDepth: 0 }); + + // cg-b is deleted while it is still the pending tail. + live = ['cg-a']; + await expect(service.runSweep()).resolves.toMatchObject({ + stale: false, + // Only the graphs that still exist contribute; the rotation still closes. + backlogDepth: 7, + }); + }); + + /** + * The per-context-graph enumeration is the call that scales with graph count, + * so it must reach the store on the background lane and carry the slice's + * deadline. An absent priority defaults to `normal`, which turned the GC's own + * discovery into a foreground scan on a cold graph-set index. + */ + it('runs the per-context-graph meta enumeration on the background lane with the slice deadline', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const seen: QueryOptions[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => ['cg-lane'], + listSharedMemoryMetaGraphs: async (_contextGraphId, options) => { + seen.push(options); + return []; + }, + }); + + await service.runSweep(); + + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ + priority: 'background', + source: 'agent.finalizedSwmCleanup.discover', + }); + expect(seen[0]!.signal).toBeInstanceOf(AbortSignal); + }); +}); diff --git a/packages/agent/vitest.unit.config.ts b/packages/agent/vitest.unit.config.ts index 130e32f77d..d25c7309c0 100644 --- a/packages/agent/vitest.unit.config.ts +++ b/packages/agent/vitest.unit.config.ts @@ -117,6 +117,7 @@ export default defineConfig({ "test/finalization-lifecycle-logger.test.ts", "test/finalized-swm-cleanup-worker.test.ts", "test/finalized-swm-cleanup-sweep.test.ts", + "test/finalized-swm-cleanup-rotation.test.ts", "test/finalization-handler.test.ts", "test/finalization-handler-chain-truth.test.ts", "test/finalization-handler-defensive-cg-id.test.ts", From d203b85235b3160cd43b633186a1e2bdf3f61e73 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:40:54 +0200 Subject: [PATCH 25/48] docs(agent): record why the cleanup task is armed after the payload write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The path-3 ordering (payload, then head, then task) was undocumented, so the next reader had no way to tell it apart from an accident. State it as what it is: removing a dependency, not closing a live race. The old order — arm, then write the payload — was already safe, because arm-and-write shared one hold of the per-KA writer lock the GC also takes, so a concurrent retireStaleTask would block and then bail on the changed write generation. But that ratchet is optional: retirement refuses to run at all when the write-gen capability is absent, so safety rested on a capability that may not be there. Arming from the head swap removes the dependency rather than relying on it, and the task is then armed only once the head matches on every field clearIfStillExact checks. Credit to the GC review for the correction: the pre-existing exposure was narrower than an earlier draft of this reasoning assumed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../agent/src/sync/requester/shared-memory-sync.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/agent/src/sync/requester/shared-memory-sync.ts b/packages/agent/src/sync/requester/shared-memory-sync.ts index 81313e9ab2..0d039221e8 100644 --- a/packages/agent/src/sync/requester/shared-memory-sync.ts +++ b/packages/agent/src/sync/requester/shared-memory-sync.ts @@ -394,6 +394,17 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // rows from the append-style metadata insert below. Otherwise // a live write could land between the swap and the append and // leave an arbitrary multi-version head. + // + // The finalized-cleanup task is armed BY this swap, i.e. after + // the payload and head are both in place, never before the + // payload. Arming first was safe only because arm-and-write + // shared one hold of this writer lock, so the GC would block + // and then bail on a changed write generation + // (`retireStaleTask`). That ratchet is optional — it refuses to + // retire when the write-gen capability is absent — so ordering + // the arm after the write removes the dependency on it instead + // of relying on it, and leaves the task armed only once the + // head matches on every field `clearIfStillExact` checks. await snapshotMaterializer.replaceHeadMetadata(pid, descriptor); for (const quad of descriptor.metadataQuads) { replacedGraphScopedMetaKeys.add(quadKey(quad)); From c02da841e1418966b1aa934dd78c3675e2211b12 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:50:51 +0200 Subject: [PATCH 26/48] test(agent): pin the finalized-cleanup re-arm on every ingest path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runSharedMemorySync reaches four outcomes inside the per-KA write lock, and only two of them call replaceHeadMetadata. The suite covered those two, so the re-arm gate could be removed wholesale and stay green: the ASK could be forced to false, or either explicit ensureFinalizedCleanupTask call deleted, and 180 tests still passed. The invariant the whole restructure rests on was guarded at one point out of four. Assert it per path, positive and negative: a finalized operation arms exactly one task bound to this kaUal, and an operation that was never finalized arms none. The negative half is not filler — arming for a live asset hands the GC a backlog entry it should never have had. Mutation-verified against the four mutants that previously survived, each applied serially with an apply-check and a git-restore proven by sha1, zero diff lines and zero leftover markers: ASK gate forced false -> 2 failed (the two non-writing paths) replaceHeadMetadata drops task -> 3 failed (both writing paths + the pre-existing late-snapshot test) superseded path drops re-arm -> 1 failed (superseded only) clean-head path drops re-arm -> 1 failed (clean head only) Each mutant kills exactly the paths it breaks and no others, which is what makes these path-specific rather than one blanket assertion. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/swm-snapshot-materializer.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index 75dc296c5d..46c8caac1f 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -533,5 +533,100 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', FINALIZED_SWM_CLEANUP_TASK_TYPE, )).toHaveLength(1); }); + + /** + * The re-arm invariant, pinned on EVERY decision path — not just the two a + * happy-path catch-up happens to take. + * + * `runSharedMemorySync` reaches four distinct outcomes inside the per-KA + * write lock, and only two of them call `replaceHeadMetadata` (which re-arms + * the GC task from the tombstone read it already performs). The other two + * write nothing at all and must re-arm via `ensureFinalizedCleanupTask`. + * Covering only the writing paths lets the whole gate be removed — the ASK + * can be forced to `false`, or either explicit call deleted — while a + * happy-path suite stays green and a finalized SWM copy is resurrected with + * no GC task to collect it. That is the exact resurrection this work exists + * to prevent, so each path is asserted on its own. + * + * The negative half matters just as much: a KA that was never finalized must + * arm NOTHING, or the GC gains a backlog entry for a live asset. + */ + describe('re-arms the finalized-cleanup GC task on every decision path', () => { + const tombstone = (operationSubject: string): Quad[] => [ + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: '"2026-07-31T10:00:00.000Z"^^', + graph: WS_META, + }, + ]; + + const armedTasks = (store: TripleStore): Promise => distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + ); + + /** + * Each seed steers the lock body to exactly ONE outcome for a served v1 + * snapshot. Keep them distinguishable: if two seeds collapsed onto the same + * branch the suite would look complete while leaving a path unguarded. + */ + const paths: Record Promise> = { + // Stored head is NEWER than the descriptor: sync must not write. + 'superseded head': async (store) => { + await store.insert([...v2.meta]); + }, + // Content already present, head rows absent: repair via replaceHeadMetadata. + 'materialized with a missing head': async (store) => { + await store.insert([ + ...v1.meta.filter((quad) => quad.subject !== v1.headSubject), + ...inGraph(v1.payload, v1.assertionGraph), + ]); + }, + // Content present and head already clean: sync must not write. + 'materialized with a clean head': async (store) => { + await store.insert([...v1.meta, ...inGraph(v1.payload, v1.assertionGraph)]); + }, + // Content absent: full replaceGraph + replaceHeadMetadata. + 'absent content': async (store) => { + await store.insert([...v1.meta]); + }, + }; + + for (const [label, seed] of Object.entries(paths)) { + it(`arms exactly one task for a finalized operation: ${label}`, async () => { + const store = new OxigraphStore(); + await seed(store); + await store.insert(tombstone(v1.operationSubject)); + + const summary = await realHarness(store, v1).run(); + + expect(summary.failedPhases).toBe(0); + const tasks = await armedTasks(store); + expect(tasks).toHaveLength(1); + // Bound to THIS lifecycle: an arbitrary task subject would satisfy a + // bare length check while pointing the GC at the wrong asset. + expect(await distinctObjects(store, WS_META, tasks[0]!, `${DKG}kaUal`)).toEqual([UAL]); + }); + + it(`arms nothing when the operation was never finalized: ${label}`, async () => { + const store = new OxigraphStore(); + await seed(store); + + const summary = await realHarness(store, v1).run(); + + expect(summary.failedPhases).toBe(0); + expect(await armedTasks(store)).toEqual([]); + }); + } + }); }); }); From 0a527181f6d5ffdf0c3099fe672512c9a5146007 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:50:56 +0200 Subject: [PATCH 27/48] test(agent): pin the late-receipt snapshot content memo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The immutable-snapshot fallback is the normal path for a late receipt once the idle GC has drained SWM, and none of its bounding behaviour had a test: a candidate cap could be reintroduced, the memo could be moved onto the throw path, or dropped entirely, with every suite green. Three properties, each pinned by the observable that actually separates it from its mutant. Search completeness: discovery orders by shareOperationId, which bears no relation to which snapshot matches, so truncating the candidate list does not do less work — it returns a different answer, and because the list is deterministic every retry re-derives the identical truncation. Five candidates with the matching one sorting last is past the boundary of the cap this replaced. Throw does not memo: failing to read one operation's snapshot says nothing about whether that content is correct, and a sibling may hold a readable copy of the same content. Two candidates share a digest, the first is made unreadable, and the second must still verify. Rejection does memo: the answer is identical whether or not repeated work is skipped, so this is asserted by counting payload resolutions. An output-equality assertion passes with the memo deleted. These target the integrated state; the search-completeness test is a regression guard that fails on the superseded candidate-cap commit. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../ka-graph-finalization-handler.test.ts | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 110aa216de..8428ab6c4e 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2699,6 +2699,180 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(0); }); + /** + * Stage extra operation subjects for the SAME assertion, so late-receipt + * discovery has several candidates to choose between. `shareOperationId` + * values are chosen to sort BEFORE the real one, because discovery is + * `ORDER BY ?shareId` and that order is unrelated to which candidate matches. + */ + async function stageDecoyOperations(input: { + count: number; + prefix: string; + contentFor: (index: number) => string; + }): Promise { + const scope = createGraphKnowledgeAssetScope(UAL, VERSION); + const shareIds: string[] = []; + for (let index = 0; index < input.count; index += 1) { + const shareOperationId = `${input.prefix}-${index}`; + shareIds.push(shareOperationId); + const value = input.contentFor(index); + await storeKnowledgeAssetOperationPublicQuads({ + store, + graphManager, + contextGraphId: CG, + shareOperationId, + kaUal: scope.ual, + assertionVersion: scope.assertionVersion, + // Same public triple count as the real assertion, so the count filter + // in discovery cannot separate them — only a full payload resolution can. + quads: [ + { subject: 'urn:asset:one', predicate: 'urn:predicate:value', object: `"${value}-one"`, graph: '' }, + { subject: 'urn:asset:two', predicate: 'urn:predicate:value', object: `"${value}-two"`, graph: '' }, + ], + publisherPeerId: '12D3KooWPublisher', + }); + } + return shareIds; + } + + async function reconcileFromImmutableSnapshot( + message: FinalizationMessageMsg, + ): Promise { + const internals = handler as unknown as { + verifyChainCgBinding: (kaId: bigint, cgId: string) => Promise; + }; + internals.verifyChainCgBinding = async () => true; + return handler.handleChainReconciledKC({ + contextGraphId: CG, + onChainCgId: '42', + ual: UAL, + merkleRoot: message.kcMerkleRoot, + publisherAddress: PUBLISHER, + kaId: PACKED_KA_ID, + versionBlock: 123, + authorAddress: AUTHOR, + // Carries no publicQuadsDigest, so discovery cannot pre-filter by content + // and every candidate must be resolved to be told apart. This is the case + // a candidate cap silently broke. + trustedAssertionEvidence: trustedRecoveryEvidence(message), + }, createOperationContext('system')); + } + + /** + * Regression guard against re-introducing a candidate cap. + * + * Discovery orders by `?shareId`, which has no relationship to which snapshot + * actually matches the receipt. Truncating that list therefore does not do + * less work, it returns a different answer — and because the list is + * deterministic, every retry re-derives the identical truncation, so the + * receipt is stranded permanently rather than delayed. Five candidates with + * the matching one last is past the boundary of the cap this replaced. + */ + it('verifies a late receipt whose matching snapshot sorts last among many candidates', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + + // 'aaa-decoy-*' sorts before 'graph-finalization-share'. + await stageDecoyOperations({ + count: 4, + prefix: 'aaa-decoy', + contentFor: (index) => `decoy-${index}`, + }); + await store.dropGraph(vmGraph); + await store.deleteByPattern({ graph: graphManager.metaGraphUri(CG), subject: UAL }); + + await expect(reconcileFromImmutableSnapshot(message)).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + }); + + /** + * The subtle half of the content memo: a THROW must not memo. + * + * Failing to READ one operation's snapshot says nothing about whether that + * content is correct, and a sibling operation may hold a readable copy of the + * very same content. Memoing on throw would therefore skip the readable copy + * and strand the receipt. Both candidates here carry identical content, so + * they share a digest — which is exactly when a wrongly-placed memo bites. + */ + it('still verifies a readable snapshot after an identical-content sibling fails to resolve', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + + // Same content as the real assertion, so it advertises the same digest, and + // it sorts first. Its snapshot payload is then made unreadable. + const scope = createGraphKnowledgeAssetScope(UAL, VERSION); + await storeKnowledgeAssetOperationPublicQuads({ + store, + graphManager, + contextGraphId: CG, + shareOperationId: 'aaa-unreadable-twin', + kaUal: scope.ual, + assertionVersion: scope.assertionVersion, + quads: [ + { subject: 'urn:asset:one', predicate: 'urn:predicate:value', object: '"one"', graph: '' }, + { subject: 'urn:asset:two', predicate: 'urn:predicate:value', object: '"two"', graph: '' }, + ], + privateMerkleRoot: message.privateMerkleRoot, + privateTripleCount: message.privateTripleCount, + publisherPeerId: '12D3KooWPublisher', + }); + const twinSnapshotGraph = (await store.query( + `SELECT ?graph WHERE { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { ` + + `<${workspaceOperationSubject(CG, 'aaa-unreadable-twin')}> ` + + ` ?graph } }`, + )); + if (twinSnapshotGraph.type !== 'bindings' || !twinSnapshotGraph.bindings[0]?.['graph']) { + throw new Error('expected a snapshot graph for the twin operation'); + } + await store.dropGraph(twinSnapshotGraph.bindings[0]['graph']!); + + await store.dropGraph(vmGraph); + await store.deleteByPattern({ graph: graphManager.metaGraphUri(CG), subject: UAL }); + + await expect(reconcileFromImmutableSnapshot(message)).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + }); + + /** + * The other half of the memo: a content rejection MUST memo, or the bound on + * repeated work does not exist and we are back to a full payload read per + * candidate. The answer is identical either way, so this can only be observed + * by counting payload resolutions — an output-equality assertion here would + * pass with the memo deleted. + */ + it('resolves an already-rejected snapshot digest only once across candidates', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + + // Two candidates with IDENTICAL wrong content, so they advertise the same + // digest and the second is provably a repeat of work already done. + await stageDecoyOperations({ count: 2, prefix: 'aaa-dup', contentFor: () => 'same-wrong' }); + await store.dropGraph(vmGraph); + await store.deleteByPattern({ graph: graphManager.metaGraphUri(CG), subject: UAL }); + + const payloadReads: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if ( + options?.source === 'agent.finalization.resolveImmutableSnapshotPayload' + && query.startsWith('CONSTRUCT') + ) payloadReads.push(query); + return originalQuery(query, options); + }); + + await expect(reconcileFromImmutableSnapshot(message)).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + // One rejected duplicate plus the matching snapshot — never the skipped twin. + expect(payloadReads).toHaveLength(2); + querySpy.mockRestore(); + }); + it('rejects mixed graph-scope and legacy-root finalization envelopes', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage( From ac2d1209cb1f65fe74c7df81eb3a0c1c71b0cc9e Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 10:56:18 +0200 Subject: [PATCH 28/48] test(agent): name two lock tests after what they verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both names asserted a property neither test checks. The first claimed to observe finalized-SWM cleanup serializing against the shared per-KA writer lock, but its blocking writer is released before the drain runs — pointing that writer at an unrelated lock key leaves the test green. The second attributed its fail-closed behaviour to the handler, when what makes it fail closed is the cleanup service being constructed without a lock map; the handler's own writeLocks option was dead and has since been deleted. A name that asserts an unchecked property is worse than no test, because it stops the next person looking for the coverage. Both still verify something worth keeping, so this renames rather than deletes and records what each actually establishes. Serialization itself is covered by the write-generation TOCTOU test, which queues the drain behind a held lock and asserts the ordering across it. Assertions and bodies are unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../ka-graph-finalization-handler.test.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 8428ab6c4e..6bdf2f3608 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2318,7 +2318,20 @@ describe('graph-scoped finalization handler', () => { })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); }); - it('serializes finalized SWM cleanup with the shared per-KA writer lock', async () => { + /** + * Despite its previous name, this test does not observe cleanup serializing + * against a contended lock: the blocking writer is released before the drain + * ever runs. Pointing that writer at an unrelated lock key leaves the test + * green, which is the proof. What it does verify is still worth keeping — + * finalization commits the VM graph even while another writer holds the + * per-KA SWM lock, and it leaves SWM entirely to the independent drain. + * + * The serialization property itself is covered by + * `re-reads the head under the lock when the store cannot track write + * generations`, which queues the drain behind a held lock and asserts the + * ordering across it. + */ + it('commits VM under a held per-KA SWM lock and leaves SWM to the later drain', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); const writeLocks = new Map>(); const lockingHandler = new FinalizationHandler(store, legacyFinalizationChain(), { @@ -2504,7 +2517,13 @@ describe('graph-scoped finalization handler', () => { })).resolves.toMatchObject({ assertionVersion: '2' }); }); - it('preserves finalized SWM when the handler has no shared writer lock', async () => { + /** + * The lock that matters here belongs to the cleanup SERVICE, not the handler: + * `cleanupService(null)` is what makes this fail closed. The uncoordinated + * handler is incidental — finalization no longer takes a per-KA SWM lock at + * all, and its `writeLocks` option has since been deleted as dead. + */ + it('preserves finalized SWM when the cleanup service has no writer lock map', async () => { const { message, swmGraph } = await stageGraph(); const uncoordinated = new FinalizationHandler(store, legacyFinalizationChain()); From 148c7a80c9a3fc341fedca8e4e302aed01afd638 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:19:37 +0200 Subject: [PATCH 29/48] test(agent): prove wake coalescing instead of cancelling it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coalescing test closed the worker immediately after releasing the sweep, so the follow-up wake was cancelled before it could be observed. Reducing the whole in-flight branch to a bare `return` left the test green; the only thing that died was a sibling asserting the yielded-slice reschedule. Coalescing has two halves and they need separate evidence. The sibling covers the re-wake half — a wake arriving during a sweep is not lost. The half asserted here is that several wakes during one sweep collapse to exactly one follow-up, which is what stops repeated wake-ups creating overlapping store scans. Nothing covered it. Three wakes now arrive mid-sweep, the sweep is released without closing, and the test asserts one scheduled follow-up rather than three, fires it, and expects exactly two sweeps total. Verified against two mutants: the bare-return branch, and an in-flight branch that schedules a timer per wake. The second is killed only by this test, which is what shows the property was previously unguarded rather than merely under-asserted. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/finalized-swm-cleanup-worker.test.ts | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/packages/agent/test/finalized-swm-cleanup-worker.test.ts b/packages/agent/test/finalized-swm-cleanup-worker.test.ts index 8be926922b..731438736d 100644 --- a/packages/agent/test/finalized-swm-cleanup-worker.test.ts +++ b/packages/agent/test/finalized-swm-cleanup-worker.test.ts @@ -32,12 +32,28 @@ describe('FinalizedSwmCleanupWorker', () => { await worker.close(); }); - it('wake is non-blocking and coalesces work while one sweep is in flight', async () => { - const timers: Array<() => void> = []; + /** + * Coalescing has two halves and they need separate evidence. + * + * The re-wake half — a wake arriving during a sweep is not lost — is already + * covered by the yielded-slice test below. The half asserted here is that + * SEVERAL wakes during one sweep collapse to exactly ONE follow-up, which is + * what stops repeated wake-ups creating overlapping store scans. + * + * The previous version of this test closed the worker immediately after + * releasing the sweep, so the follow-up was cancelled before it could be + * observed and the whole in-flight branch could be reduced to a bare `return` + * with this test still green. It now lets the follow-up schedule and fire. + */ + it('collapses several wakes during one sweep into exactly one follow-up sweep', async () => { + const timers: Array<{ fn: () => void; delayMs: number }> = []; let release!: () => void; const pending = new Promise((resolve) => { release = resolve; }); + let started = 0; const sweep = vi.fn(async () => { - await pending; + started += 1; + // Only the first pass blocks; the follow-up must be free to complete. + if (started === 1) await pending; return { backlogDepth: 0, oldestMarkerAt: null, @@ -47,8 +63,8 @@ describe('FinalizedSwmCleanupWorker', () => { }); const worker = new FinalizedSwmCleanupWorker({ sweep, - setTimer: ((fn: () => void) => { - timers.push(fn); + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); return { unref() {} }; }) as never, clearTimer: () => {}, @@ -56,14 +72,33 @@ describe('FinalizedSwmCleanupWorker', () => { expect(worker.wake()).toBeUndefined(); expect(sweep).not.toHaveBeenCalled(); - timers.shift()!(); + timers.shift()!.fn(); await Promise.resolve(); expect(sweep).toHaveBeenCalledTimes(1); + + // Three wakes while one sweep is in flight. None may start a sweep, and + // none may schedule a timer of its own. expect(worker.wake()).toBeUndefined(); expect(worker.wake()).toBeUndefined(); + expect(worker.wake()).toBeUndefined(); + expect(sweep).toHaveBeenCalledTimes(1); + expect(timers).toHaveLength(0); + release(); - await worker.close(); + await new Promise((resolve) => { setImmediate(resolve); }); + + // Exactly one follow-up was scheduled for the three coalesced wakes. expect(sweep).toHaveBeenCalledTimes(1); + expect(timers).toHaveLength(1); + + timers.shift()!.fn(); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(sweep).toHaveBeenCalledTimes(2); + // A clean second pass asks for nothing further. + expect(timers).toHaveLength(0); + + await worker.close(); + expect(sweep).toHaveBeenCalledTimes(2); }); it('contains sweep errors and exposes the last failure', async () => { From 067e06f1eb7fd2a2b019c759275870ab03603140 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:21:40 +0200 Subject: [PATCH 30/48] fix(agent): keep thrown scheduler pressure on the GC retry path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runSweep let StoreSchedulerBusyError escape to the worker's catch block, which records lastError and calls onError but never schedules a retry — retry scheduling exists only on the success path. So pressure arriving as a throw rather than through the point-in-time snapshot gate stranded the backlog until the 15-minute periodic backstop, with the 5s retry delay sitting unused. Reachable at defaults: discovery runs on the background lane, which has an in-flight ceiling of 1 and a 10s queue-wait timeout, so a queue_wait_timeout rejection is an expected outcome under load, and the snapshot gate cannot see load that lands between the gate and the query. Classify it in the service, at the four call sites that already yield the slice on abort, rather than in the worker. A scheduler rejection is store pressure reported by the lane itself, so `pressureSkipped` — which already means transient-and-retryable — is the honest classification, and it makes `pressureSkips` and the stale backlog counters behave exactly as they do for gate-detected pressure instead of blanking. Yielding through `yieldRotation` also preserves the rotation cursor, so the sweep resumes where it stopped rather than restarting the rotation. The worker stays dependency-free; recognising the error class there would have coupled a pure scheduler to the storage package. Unknown errors still throw: an unrecognised fault must not hot-retry every few seconds, and the periodic backstop is the right cadence for it. `cleanupKnownMetaGraph` is untouched and still propagates the throw, which a test asserts. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../agent/src/finalized-swm-cleanup-service.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index 8c35e5c891..0de24ddab0 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -181,6 +181,15 @@ export class FinalizedSwmCleanupService { contextGraphIds = await this.listContextGraphIds(discoveryOptions); } catch (error) { if (deadlineSignal.aborted) return deferredResult(0, 'budget'); + // A scheduler rejection IS store pressure — the background lane reporting + // that it is saturated — it just arrives as a throw instead of through the + // point-in-time snapshot gate, which cannot see load that lands between the + // gate and the query. Classifying it keeps the sweep on the retry path and + // counted in `pressureSkips`; letting it escape strands the backlog until + // the periodic backstop for a transient, retryable reason. Every other + // error still throws: an unknown fault must not hot-retry every few + // seconds, and the backstop is the right cadence for it. + if (error instanceof StoreSchedulerBusyError) return deferredResult(0, 'pressure'); throw error; } const pending = this.resumeRotation(contextGraphIds); @@ -200,6 +209,7 @@ export class FinalizedSwmCleanupService { metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId, discoveryOptions); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); + if (error instanceof StoreSchedulerBusyError) return yieldRotation('pressure'); throw error; } // A context graph contributes to the rotation total only once every one of @@ -221,6 +231,7 @@ export class FinalizedSwmCleanupService { }); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); + if (error instanceof StoreSchedulerBusyError) return yieldRotation('pressure'); throw error; } deletedItems += cleanup.deletedItems; @@ -238,6 +249,9 @@ export class FinalizedSwmCleanupService { backlog = await this.inspectBacklog(swmMetaGraph, deadlineSignal); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); + // Also covers this method's own defensive pressure throw, which the + // snapshot gate above races with rather than prevents. + if (error instanceof StoreSchedulerBusyError) return yieldRotation('pressure'); throw error; } contextGraphBacklogDepth += backlog.depth; From 8f565739ec40ff00ee5b10717f732a02a8dcb966 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:27:17 +0200 Subject: [PATCH 31/48] refactor(agent): name the transient-vs-terminal sweep failure policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retry classification read as four inline `instanceof` checks, which states the type but not the policy. The decision being made is whether a failure is transient (must stay on the short retry path or the backlog strands) or terminal (must not, or a broken sweep spins every few seconds forever). Extract `isTransientStorePressure` so that distinction has one name, one place for the rationale, and one place to change — widening it is equivalent to adding a retry and should get the same scrutiny. Behaviour unchanged. The `instanceof` inside cleanupMetaGraph's row loop stays as-is: that one is loop control, not failure classification. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index 0de24ddab0..276dc19ce1 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -62,6 +62,23 @@ function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): bo ].some((count) => count > 0); } +/** + * Split sweep failures into transient and terminal, because the two need + * opposite scheduling. A scheduler rejection is the store's own lane reporting + * saturation — the same condition {@link hasActiveStorePressure} samples, only + * raised as a throw because the load arrived after the gate rather than before + * it. Those must stay on the short retry path or the backlog strands until the + * periodic backstop. + * + * Everything else is terminal until proven otherwise and must NOT be retried on + * that cadence: a genuinely broken sweep would spin every few seconds forever, + * trading a stranded backlog for a hot loop. Widening this predicate therefore + * needs the same scrutiny as adding a retry. + */ +function isTransientStorePressure(error: unknown): boolean { + return error instanceof StoreSchedulerBusyError; +} + export interface FinalizedSwmCleanupServiceOptions { store: TripleStore; writeLocks?: Map>; @@ -189,7 +206,7 @@ export class FinalizedSwmCleanupService { // the periodic backstop for a transient, retryable reason. Every other // error still throws: an unknown fault must not hot-retry every few // seconds, and the backstop is the right cadence for it. - if (error instanceof StoreSchedulerBusyError) return deferredResult(0, 'pressure'); + if (isTransientStorePressure(error)) return deferredResult(0, 'pressure'); throw error; } const pending = this.resumeRotation(contextGraphIds); @@ -209,7 +226,7 @@ export class FinalizedSwmCleanupService { metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId, discoveryOptions); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); - if (error instanceof StoreSchedulerBusyError) return yieldRotation('pressure'); + if (isTransientStorePressure(error)) return yieldRotation('pressure'); throw error; } // A context graph contributes to the rotation total only once every one of @@ -231,7 +248,7 @@ export class FinalizedSwmCleanupService { }); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); - if (error instanceof StoreSchedulerBusyError) return yieldRotation('pressure'); + if (isTransientStorePressure(error)) return yieldRotation('pressure'); throw error; } deletedItems += cleanup.deletedItems; @@ -251,7 +268,7 @@ export class FinalizedSwmCleanupService { if (deadlineSignal.aborted) return yieldRotation('budget'); // Also covers this method's own defensive pressure throw, which the // snapshot gate above races with rather than prevents. - if (error instanceof StoreSchedulerBusyError) return yieldRotation('pressure'); + if (isTransientStorePressure(error)) return yieldRotation('pressure'); throw error; } contextGraphBacklogDepth += backlog.depth; From 4b49265fb59239b4197458af208d766dc2147c8d Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:28:31 +0200 Subject: [PATCH 32/48] test(agent): pin scheduler-rejection retry and its boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retry scheduling lives only on the worker's success path, so a store rejection thrown out of the sweep skipped it entirely and stranded the backlog until the periodic backstop. The service now classifies that class as pressure; nothing pinned either the classification or its limit. Both tests wire the real service into the real worker so the rejection travels the path it travels in production, rather than stubbing the sweep and asserting the worker's own arithmetic back at itself. The positive asserts the prompt retry, that the deferral is counted rather than swallowed, and that it is not reported as a fault. The negative asserts an unknown failure is still not retried: broadening the classification to every throw would make a genuine fault hot-retry every few seconds forever, and the positive alone would still pass. That boundary is the whole reason the classification is worth having. The seam is the backlog probe, which raises this class itself as a defensive check immediately after the snapshot gate — so the service can manufacture the error with no scheduler load at all, only the race between the gate and the check. Verified against three mutants: classification removed, classification broadened to every error, and all four sites removed. The middle one is killed only by the negative test. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/finalized-swm-cleanup-sweep.test.ts | 95 ++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/packages/agent/test/finalized-swm-cleanup-sweep.test.ts b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts index 46df8742be..0a8dbd9a1e 100644 --- a/packages/agent/test/finalized-swm-cleanup-sweep.test.ts +++ b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts @@ -19,7 +19,12 @@ */ import { describe, expect, it, vi } from 'vitest'; -import { GraphManager, OxigraphStore, type TripleStore } from '@origintrail-official/dkg-storage'; +import { + GraphManager, + OxigraphStore, + StoreSchedulerBusyError, + type TripleStore, +} from '@origintrail-official/dkg-storage'; import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; import { FinalizedSwmCleanupWorker } from '../src/finalized-swm-cleanup-worker.js'; @@ -387,4 +392,92 @@ describe('finalized SWM cleanup sweep gates', () => { expect(timers[0]!.delayMs).toBe(321); await worker.close(); }); + + /** + * Wires the real service into the real worker so a store rejection travels + * the whole path it travels in production. + * + * The snapshot gate is a point-in-time sample and cannot see load that lands + * between the gate and the query, so the background lane reports saturation + * by THROWING instead. `inspectBacklog` even raises that class itself as a + * defensive check straight after the gate, so the service can manufacture the + * error without any scheduler load at all — just the race between the two. + * + * Before this was classified, such a throw skipped the worker's retry + * scheduling entirely (which exists only on the success path) and stranded + * the backlog until the 15-minute backstop. + */ + function schedulerThrowFixture(error: unknown) { + const store = new OxigraphStore(); + installPressureSwitch(store); + const originalQuery = store.query.bind(store); + vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.backlog') throw error; + return originalQuery(query, options); + }); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + const timers: Array<{ fn: () => void; delayMs: number }> = []; + const onError = vi.fn(); + const worker = new FinalizedSwmCleanupWorker({ + sweep: () => service.runSweep(), + retryDelayMs: 5_000, + onError, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + return { worker, timers, onError }; + } + + it('retries and counts a scheduler rejection thrown out of the sweep as pressure', async () => { + const { worker, timers, onError } = schedulerThrowFixture( + new StoreSchedulerBusyError('queue_full', 'background', 'agent.finalizedSwmCleanup.backlog'), + ); + + await worker.runNow(); + + // Retried on the prompt cadence rather than left to the periodic backstop. + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(5_000); + // Counted as a deferral, not swallowed as a fault: the counters must stay + // meaningful exactly when deferral is happening. + expect(worker.snapshot()).toMatchObject({ + runs: 1, + pressureSkips: 1, + lastError: null, + backlogStale: true, + }); + expect(onError).not.toHaveBeenCalled(); + await worker.close(); + }); + + /** + * The negative that makes the classification mean something. Broadening it to + * catch every throw would make an unknown fault hot-retry every few seconds + * forever, and the positive test above would still pass — so the boundary + * needs its own assertion. + */ + it('does not retry an unknown sweep failure', async () => { + const { worker, timers, onError } = schedulerThrowFixture(new Error('store exploded')); + + await worker.runNow(); + + expect(timers).toHaveLength(0); + expect(onError).toHaveBeenCalledTimes(1); + expect(worker.snapshot()).toMatchObject({ + runs: 1, + pressureSkips: 0, + lastError: 'store exploded', + }); + await worker.close(); + }); }); From 70322bd7158cc589c1512a99cc248791d67cc5a2 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:37:48 +0200 Subject: [PATCH 33/48] test(agent): cover each scheduler-rejection site independently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transient/terminal classification exists at four separate await boundaries in a sweep, and one end-to-end test covered only whichever one it happened to trigger: deleting the conversion at context-graph enumeration, meta-graph enumeration or candidate discovery left the suite green. One test standing in for four sites is the shape that makes coverage look better than it is. Each boundary is now driven separately, in both directions: a scheduler rejection there is a retryable deferral, and an unknown failure there stays terminal. The negatives are what stop the classification widening into a blanket retry, which would hot-retry a genuine fault forever. The two enumeration boundaries differ from the two query boundaries in what they must preserve — the context-graph enumeration fails before a rotation exists, so it defers outright, while the later three yield the rotation and keep the cursor. A converted rejection therefore also has to behave like a pressure yield rather than merely return the right shape: the interrupted context graph is re-measured whole next slice, publishing two markers rather than three, so the error path cannot reintroduce the double-count the accumulator exists to prevent. Verified against six mutants: each of the four conversion sites removed individually, and the shared predicate forced true and false. Every site mutant now kills its own case; before, three of the four survived. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/finalized-swm-cleanup-sweep.test.ts | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/packages/agent/test/finalized-swm-cleanup-sweep.test.ts b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts index 0a8dbd9a1e..cc7dba1758 100644 --- a/packages/agent/test/finalized-swm-cleanup-sweep.test.ts +++ b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts @@ -480,4 +480,133 @@ describe('finalized SWM cleanup sweep gates', () => { }); await worker.close(); }); + + /** + * The classification exists at FOUR independent `await` boundaries inside a + * sweep, and a single end-to-end test covers only whichever one it happens to + * trigger — the other three can be deleted with the suite green. Each is + * therefore driven separately here. + * + * The two enumeration boundaries and the two query boundaries also differ in + * what they must preserve: the context-graph enumeration fails before a + * rotation exists, so it defers outright, while the later three must yield + * the rotation and keep the cursor. + */ + type PressureSite = 'context-graph enumeration' | 'meta-graph enumeration' + | 'candidate discovery' | 'the backlog probe'; + + function serviceThrowingAt(site: PressureSite, error: unknown) { + const store = new OxigraphStore(); + installPressureSwitch(store); + const metaGraph = new GraphManager(store).sharedMemoryMetaUri(CG); + const originalQuery = store.query.bind(store); + vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (site === 'candidate discovery' && options?.source === 'agent.finalizedSwmCleanup.discover') { + throw error; + } + if (site === 'the backlog probe' && options?.source === 'agent.finalizedSwmCleanup.backlog') { + throw error; + } + return originalQuery(query, options); + }); + return new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => { + if (site === 'context-graph enumeration') throw error; + return [CG]; + }, + listSharedMemoryMetaGraphs: async () => { + if (site === 'meta-graph enumeration') throw error; + return [metaGraph]; + }, + }); + } + + const PRESSURE_SITES: PressureSite[] = [ + 'context-graph enumeration', + 'meta-graph enumeration', + 'candidate discovery', + 'the backlog probe', + ]; + + it.each(PRESSURE_SITES)( + 'classifies a scheduler rejection raised at %s as a retryable deferral', + async (site) => { + const service = serviceThrowingAt( + site, + new StoreSchedulerBusyError('queue_full', 'background', 'agent.finalizedSwmCleanup'), + ); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + stale: true, + deletedItems: 0, + }); + }, + ); + + it.each(PRESSURE_SITES)( + 'lets an unknown failure at %s stay terminal', + async (site) => { + const service = serviceThrowingAt(site, new Error(`unknown failure at ${site}`)); + + await expect(service.runSweep()).rejects.toThrow(`unknown failure at ${site}`); + }, + ); + + /** + * A converted throw must behave like a pressure yield, not merely return the + * right shape: the context graph it interrupted has to be re-measured whole + * on the next slice. Committing its partial contribution would double-count + * it on resume — the same defect the rotation accumulator exists to prevent, + * arriving through the error path instead of the gate. + */ + it('re-measures an interrupted context graph whole after a converted rejection', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const rootMeta = graphManager.sharedMemoryMetaUri(CG); + const subMeta = graphManager.sharedMemoryMetaUri(CG, 'named'); + const marker = (label: string, graph: string) => ([ + { subject: `urn:dkg:finalized-swm-cleanup:${label}`, predicate: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', object: 'http://dkg.io/ontology/FinalizedSwmCleanupTask', graph }, + { subject: `urn:dkg:finalized-swm-cleanup:${label}`, predicate: 'http://dkg.io/ontology/finalizedSwmCleanupRoot', object: JSON.stringify(`0x${'11'.repeat(32)}`), graph }, + ]); + await store.insert([...marker('a', rootMeta), ...marker('b', subMeta)]); + + let backlogReads = 0; + const originalQuery = store.query.bind(store); + vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.backlog') { + backlogReads += 1; + // Interrupt AFTER the first meta graph has been measured, so a + // partial contribution exists to be wrongly committed. + if (backlogReads === 2) { + throw new StoreSchedulerBusyError('queue_full', 'background', 'agent.finalizedSwmCleanup.backlog'); + } + } + return originalQuery(query, options); + }); + + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [rootMeta, subMeta], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + stale: true, + // The one marker measured before the throw is NOT published. + backlogDepth: 0, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + stale: false, + // Two markers total — the interrupted graph re-measured whole, not three. + backlogDepth: 2, + }); + }); }); From 6656dc28562112310f4163b3bc49d8507ca8bfa2 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:46:11 +0200 Subject: [PATCH 34/48] test(agent): pin all three local-cleanup predicates against peer leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LOCAL_FINALIZED_SWM_CLEANUP_PREDICATES and the store-side row filter both list three predicates, but the responder fixtures only ever carried finalizedSwmCleanupRoot. finalizedSwmCleanupMarkedAt and finalizedSwmCleanupHeadFingerprint appeared zero times, so dropping either from the filter leaked it to peers with every test still green. One predicate was standing in for three. The property is peer leak — local GC bookkeeping reaching other nodes — so seed the rows where they can actually escape. The subject-level FILTER NOT EXISTS strips whole cleanup-task subjects, which means a predicate only reachable on a task subject cannot demonstrate the predicate guard at all. markFinalizedGraphScopedSwmForCleanup writes root and markedAt onto the OPERATION subject, which is served, so that is where the fixture puts them. The head fingerprint is not written there today; it is seeded anyway so the guard is pinned as load-bearing rather than incidental, and a future writer cannot leak it by attaching it to a served subject. Assert all three by name across the TTL-disabled, TTL-enabled and session-scoped readers, plus the post-cleanup response, and assert the positive: normal lifecycle rows must still be served, or filtering everything would satisfy the absence checks. Mutation-verified, chain-free, each restored to a zero-line diff: markedAt removed from both layers -> killed (2 failed) headFingerprint removed from both layers -> killed (2 failed) markedAt removed store-side only -> killed (2 failed) headFingerprint removed store-side only -> killed (2 failed) markedAt removed in-process only -> SURVIVED The last one is structural, not a gap left open: every lane also filters at the store, so the in-process set is defence-in-depth that no test can pin while the store-side filter stands. Pinning it would mean weakening the store-side clause, which is load-bearing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/sync-responder-swm-subgraphs.test.ts | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index ced58f9de8..1c7feeb200 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -420,6 +420,7 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { const cleanupTask = 'urn:dkg:finalized-swm-cleanup:responder-test'; const head = `${ual}#dkg-swm-head`; const assertionGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ab/9`; + const markedAt = new Date(Date.now() - 2_000).toISOString(); const markedEntity = 'urn:swm:finalized:still-syncable'; const unmarkedGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ac/10`; const unmarkedRoot = 'urn:swm:unmarked:must-sync'; @@ -440,8 +441,25 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, { graph: ROOT_SWM_META, subject: cleanupTask, predicate: RDF_TYPE, object: `${DKG_NS}FinalizedSwmCleanupTask` }, { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, + // The task subject carries all three local predicates, exactly as + // buildFinalizedSwmCleanupTaskQuads writes them. { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupMarkedAt`, object: `"${markedAt}"^^` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupHeadFingerprint`, object: `"${'cd'.repeat(32)}"` }, + // The OPERATION subject is the one that matters for the predicate + // guard. It is served to peers, so the subject-level + // `FILTER NOT EXISTS { ?s a FinalizedSwmCleanupTask }` does not cover + // it and only the predicate list keeps these rows local. + // + // Root and markedAt are what markFinalizedGraphScopedSwmForCleanup + // actually writes here (finalization-handler.ts:1602-1612). The head + // fingerprint is NOT written on an operation subject today — it is + // seeded anyway so the predicate guard is pinned as load-bearing + // rather than incidental: a future writer must not be able to leak it + // by putting it on a subject the responder serves. { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupMarkedAt`, object: `"${markedAt}"^^` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupHeadFingerprint`, object: `"${'cd'.repeat(32)}"` }, { graph: assertionGraph, subject: markedEntity, predicate: 'http://schema.org/name', object: '"finalized-copy"' }, { graph: unmarkedGraph, subject: unmarkedRoot, predicate: 'http://schema.org/name', object: '"live-copy"' }, { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, @@ -471,10 +489,39 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { phase: 'meta', }); + // Every local predicate, asserted by name. One predicate standing in + // for three is how two thirds of this guard went unheld: the fixtures + // only ever carried `finalizedSwmCleanupRoot`, so dropping either other + // entry from the filter leaked it to peers with the suite still green. + const expectNoLocalGcLeak = (payload: string) => { + expect(payload).not.toContain('finalizedSwmCleanupRoot'); + expect(payload).not.toContain('finalizedSwmCleanupMarkedAt'); + expect(payload).not.toContain('finalizedSwmCleanupHeadFingerprint'); + expect(payload).not.toContain(cleanupTask); + }; + expect(out).toContain(head); expect(out).toContain(op); - expect(out).not.toContain(cleanupTask); - expect(out).not.toContain('finalizedSwmCleanupRoot'); + // Positive half: filtering everything would satisfy the absence checks. + expect(out).toContain(`${DKG_NS}shareOperationId`); + expect(out).toContain(`${DKG_NS}assertionGraph`); + expectNoLocalGcLeak(out); + + // The TTL-disabled lane serves two different readers: without a session + // it store-pages via readSwmMetaRowsPage, with one it loads the bounded + // snapshot. They filter in different places — store-side SPARQL vs the + // in-process strip — so a per-predicate hole can exist in one and not + // the other. + const sessionOut = await markedCap.invoke({ + contextGraphId: CG_ID, + syncSessionId: `finalized-cleanup-session-${sharedMemoryTtlMs}`, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + expect(sessionOut).toContain(op); + expectNoLocalGcLeak(sessionOut); const dataOut = await markedCap.invoke({ contextGraphId: CG_ID, @@ -508,7 +555,7 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { phase: 'meta', }); expect(afterCleanup).toContain(op); - expect(afterCleanup).not.toContain('finalizedSwmCleanupRoot'); + expectNoLocalGcLeak(afterCleanup); await markedStore.close(); }, ); From dfb8bd7e88b445e6a1faebc3a243c6ff00dae6a0 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 11:59:17 +0200 Subject: [PATCH 35/48] test(agent): pin each local-cleanup predicate in its own case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit asserted all three predicates inside one parametrized case. That proves none of them leaks, but it cannot show the coverage DISCRIMINATES: every single-predicate mutant killed the same two cases, and an identical kill set is equally consistent with one assertion doing all the work. Same shape as the bug being fixed — one fixture standing in for three predicates. Split them, so each mutant reddens cases the other two leave green. Kill sets, captured by test name: drop finalizedSwmCleanupRoot -> 2 shared + the 2 root cases drop finalizedSwmCleanupMarkedAt -> 2 shared + the 2 markedAt cases drop finalizedSwmCleanupHeadFingerprint -> 2 shared + the 2 fingerprint cases The lanes are pinned the same way. Removing the filter from ONE reader reddens only that reader's cases, which is what shows a per-lane regression actually reaches the suite rather than being masked by a sibling lane: readSwmMetaRowsPage (legacy, TTL-disabled) -> only ttl=0 cases readFreshSwmMetaRowsPageFromPlan (TTL lane) -> only ttl=5000 cases Removing the predicate from the in-process set alone still SURVIVES, as before: every lane also filters at the store, so that set is defence in depth no test can pin without weakening the store-side clause. Also fixes the it.each title, which printed its arguments in the wrong order and labelled the predicate as the TTL. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/sync-responder-swm-subgraphs.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index 1c7feeb200..a19a6b77b7 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -559,6 +559,84 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { await markedStore.close(); }, ); + + /** + * One assertion per predicate, so the three entries in the local-cleanup + * filter are pinned INDEPENDENTLY. + * + * The test above asserts all three inside one case, which proves none of + * them leaks but cannot show that the coverage discriminates: every + * single-predicate mutant kills exactly the same two cases, so an identical + * kill set is equally consistent with one assertion doing all the work. + * Splitting per predicate makes the kill sets differ — dropping + * `finalizedSwmCleanupMarkedAt` from the filter reddens only the markedAt + * rows, and likewise for the other two. + * + * Seeded on the OPERATION subject deliberately. The subject-level + * `FILTER NOT EXISTS { ?s a FinalizedSwmCleanupTask }` removes whole task + * subjects, so a predicate placed there is stripped whatever the predicate + * list says, and the assertion would hold with the entry deleted. + */ + it.each([ + ['finalizedSwmCleanupRoot', 0], + ['finalizedSwmCleanupMarkedAt', 0], + ['finalizedSwmCleanupHeadFingerprint', 0], + ['finalizedSwmCleanupRoot', 5_000], + ['finalizedSwmCleanupMarkedAt', 5_000], + ['finalizedSwmCleanupHeadFingerprint', 5_000], + ] as const)( + 'never advertises the %s local cleanup predicate on a served subject (ttl=%s)', + async (leakedPredicate, sharedMemoryTtlMs) => { + const store = new OxigraphStore(); + const publishedAt = new Date(Date.now() - 1_000).toISOString(); + const ual = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ad/11'; + const opId = 'per-predicate-leak'; + const op = `urn:dkg:share:${CG_ID}:${opId}`; + const assertionGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ad/11`; + const rootEntity = 'urn:swm:per-predicate:root'; + await store.insert([ + { graph: ROOT_SWM_META, subject: op, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}rootEntity`, object: rootEntity }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}contentScopeVersion`, object: '"2"^^' }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}kaUal`, object: ual }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}assertionVersion`, object: '"11"^^' }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}shareOperationId`, object: `"${opId}"` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}${leakedPredicate}`, object: `"local-only-${leakedPredicate}"` }, + { graph: assertionGraph, subject: rootEntity, predicate: 'http://schema.org/name', object: '"served-payload"' }, + ]); + const cap = captureHandler(); + registerSyncHandler({ + register: cap.register, + protocolSync: '/origintrail/dkg/sync/1.0.0', + syncDeniedResponse: 'sync-denied', + syncPageSize: 5000, + sharedMemoryTtlMs, + store, + peerId: 'self-peer', + parseSyncRequest: (data) => JSON.parse(new TextDecoder().decode(data)) as SyncRequestEnvelope, + authorizeSyncRequest: async () => true, + logWarn: noopLog, + logDebug: noopLog, + }); + + const out = await cap.invoke({ + contextGraphId: CG_ID, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + + // Positive first: the subject IS served, so absence of the predicate + // cannot be explained by the whole subject being filtered out. + expect(out).toContain(op); + expect(out).toContain(`${DKG_NS}shareOperationId`); + expect(out).not.toContain(leakedPredicate); + expect(out).not.toContain(`local-only-${leakedPredicate}`); + await store.close(); + }, + ); }); // Codex review on #885 — URI shape alone is NOT a sufficient CG From b7b4b4bb169e8173748000e43f4ebf7f65cf8c8c Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 12:10:01 +0200 Subject: [PATCH 36/48] fix(agent): derive responder cleanup IRIs from the canonical constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The responder re-typed the finalized-cleanup predicate and task-type IRIs as local literals off its own DKG prefix, while the marker writer imports them from dkg-agent-constants. The values match today — verified — so this is drift risk, not a live leak. It is worth closing because of where it sits. If a predicate IRI ever changed, the writer would follow and the responder would keep filtering the old string, so local GC bookkeeping would be advertised to peers, silently. And the mutation matrix on this filter shows in-process-only drift SURVIVES testing: every lane also filters at the store, so no test can catch it. An unguarded silent peer-leak path is worth four lines. Derive all four, keeping the short aliases so the SPARQL templates stay readable. graph-plan.ts already imported from dkg-agent-constants, so this adds names to an existing import rather than a new module edge. Behaviour is unchanged and measured, not argued: the same store-side mutants kill the same cases through the indirection, per predicate, and the task-type clause mutant produces a result identical to the pre-change baseline. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../agent/src/sync/responder/graph-plan.ts | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 8b848af0ec..d7848e12a4 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -30,7 +30,13 @@ import { type SyncResponderSnapshotBudget, } from './snapshot-budget.js'; import { estimateStringRowHeapBytes } from '../memory-telemetry.js'; -import { SYNC_BYTE_BUDGET_RESPONSE_BYTES } from '../../dkg-agent-constants.js'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, + SYNC_BYTE_BUDGET_RESPONSE_BYTES, +} from '../../dkg-agent-constants.js'; import type { ChangelogSyncResponse, ChangelogDeltaRecord } from '../changelog/wire.js'; import { durableMetaDelegationSubjectAdmissionExpression } from './durable-meta-admission.js'; import { exactAssetFilterKey } from '../exact-assets.js'; @@ -53,10 +59,17 @@ const DKG_CONTENT_SCOPE_VERSION = `${DKG}contentScopeVersion`; const DKG_KA_UAL = `${DKG}kaUal`; const DKG_ASSERTION_VERSION = `${DKG}assertionVersion`; const DKG_SHARE_OPERATION_ID = `${DKG}shareOperationId`; -const DKG_FINALIZED_SWM_CLEANUP_ROOT = `${DKG}finalizedSwmCleanupRoot`; -const DKG_FINALIZED_SWM_CLEANUP_MARKED_AT = `${DKG}finalizedSwmCleanupMarkedAt`; -const DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT = `${DKG}finalizedSwmCleanupHeadFingerprint`; -const DKG_FINALIZED_SWM_CLEANUP_TASK = `${DKG}FinalizedSwmCleanupTask`; +// DERIVED from the canonical constants the marker WRITER uses, never retyped. +// These four decide what stays local: three predicates the row filter strips, +// and the task type the subject-level FILTER NOT EXISTS matches. A local +// literal that drifted from the writer would keep filtering the old IRI and +// silently advertise local GC bookkeeping to peers — and per the mutation +// matrix on this filter, in-process-only drift survives testing, so nothing +// would catch it. Short aliases only, so the SPARQL templates stay readable. +const DKG_FINALIZED_SWM_CLEANUP_ROOT = FINALIZED_SWM_CLEANUP_ROOT_PREDICATE; +const DKG_FINALIZED_SWM_CLEANUP_MARKED_AT = FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE; +const DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT = FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE; +const DKG_FINALIZED_SWM_CLEANUP_TASK = FINALIZED_SWM_CLEANUP_TASK_TYPE; /** In-process mirror of the store-side predicate filter below. */ const LOCAL_FINALIZED_SWM_CLEANUP_PREDICATES = new Set([ DKG_FINALIZED_SWM_CLEANUP_ROOT, From 4579522e72612f1e7a252b61209c4f2575d187b4 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:17:58 +0200 Subject: [PATCH 37/48] fix(agent): close the rotation when its sole head cannot finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A yield inside a context graph's meta-graph loop leaves that graph at the head of `rotationPending`. Once every other graph has completed, the pending list is a single element, and the stall escape at resumeRotation required `pending.length > 1` — correct in itself, since rotating a one-element list is a no-op, but it left no escape at all for a sole head. `rotationPending` then never returned to null, so the rotation never re-seeded, and every OTHER context graph on the node was never swept again. Because the cursor is in-memory, that persisted until restart: #1996 reintroduced node-wide by the subsystem that exists to close it. Reproduced against the compiled service (3 CGs, slow one last with 12 meta graphs, 120ms budget): slices 1-7 entered only the slow graph and the other two were served 0 times. After the fix they are served on every third slice as the rotation re-seeds. On a sole stalled head, close the rotation and re-seed from the current context-graph list. The partial accumulator is discarded rather than published: an incomplete rotation must never become a whole-node total, so `lastKnownBacklogDepth` keeps the last complete measurement and slices keep reporting `stale` until a rotation genuinely finishes. A node whose budget cannot fit one context graph reports an honestly unknown backlog instead of a confident wrong one. The warning now also fires on this path. Previously it lived inside the deferral that could not trigger for a sole head, so the one case that stranded the node was the one case that logged nothing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index 276dc19ce1..0db557aa0e 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -326,9 +326,11 @@ export class FinalizedSwmCleanupService { * drained signal it exists to carry. * * Guarantees forward progress: after two consecutive slices fail to finish the - * head, the third defers it to the back of the rotation so the context graphs - * behind it are still served. It keeps its place in the rotation and is - * retried there. + * head, the third either defers it to the back of the rotation, so the context + * graphs behind it are still served, or — when it is the only one left, where + * deferring is a no-op — closes the rotation and re-seeds, so the rest of the + * node is swept again. Either way the stalled graph keeps its place and is + * retried; neither path drops it. */ private resumeRotation(contextGraphIds: string[]): string[] { let pending: string[]; @@ -348,13 +350,36 @@ export class FinalizedSwmCleanupService { this.rotationHeadContextGraphId = head; this.rotationHeadStalledSweeps = 0; } - if (this.rotationHeadStalledSweeps >= 2 && pending.length > 1) { + if (this.rotationHeadStalledSweeps >= 2) { + const soleHead = pending.length === 1; this.log.warn( createOperationContext('system'), - `Deferring context graph ${head} in finalized-SWM cleanup rotation; ` - + `${this.rotationHeadStalledSweeps} consecutive slices could not finish it`, + `Context graph ${head} was not finished by ${this.rotationHeadStalledSweeps} ` + + `consecutive finalized-SWM cleanup slices; ` + + (soleHead + ? 'closing the rotation early so every other context graph is swept again' + : 'deferring it to the back of the rotation'), ); - pending = [...pending.slice(1), pending[0]!]; + if (soleHead) { + // Rotating a one-element list is a no-op, so without this the rotation + // could never close: `rotationPending` would stay non-null forever, the + // re-seed below would never run, and every OTHER context graph on the + // node would never be swept again — #1996 reintroduced node-wide by the + // subsystem that exists to close it. + // + // The partial accumulator is discarded rather than published. An + // incomplete rotation must never become a whole-node total, so + // `lastKnownBacklogDepth` keeps the last complete measurement and every + // slice keeps reporting `stale` until a rotation genuinely finishes. + // A node whose budget cannot fit one context graph therefore reports an + // honestly unknown backlog, and this warning names the graph to raise it + // for. + this.rotationBacklogDepth = 0; + this.rotationOldestMarkerAt = null; + pending = [...contextGraphIds]; + } else { + pending = [...pending.slice(1), pending[0]!]; + } this.rotationHeadContextGraphId = pending[0] ?? null; this.rotationHeadStalledSweeps = 0; } From df43881ec5679aab3729dcec4194288eee344479 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:21:12 +0200 Subject: [PATCH 38/48] fix(agent): resume the meta-graph walk inside an oversized context graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closing the rotation for a sole stalled head restores node-wide progress, but the stalled graph itself still restarted its meta-graph walk at index 0 every slice and yielded at the same point. Everything past that point was never cleaned — the rotation cursor's own failure mode, one level down, and #1996 for those meta graphs. Measured on the repro (12 meta graphs, 120ms budget): 4 of 12 distinct meta graphs were ever reached, no matter how many slices ran. Now 12 of 12. Carry a per-graph resume offset and rotate the walk by it, wrapping. The walk still covers every meta graph in one pass, so a context graph still contributes to the rotation total only when fully measured and the no- double-count invariant is untouched — only the starting point moves. The cursor clears whenever the graph completes, so a graph that fits in one slice always starts at the top. Separate from the sole-head fix so it can be dropped independently: that one closes a node-wide strand, this one a per-graph one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index 0db557aa0e..f3d942d55c 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -139,6 +139,15 @@ export class FinalizedSwmCleanupService { */ private rotationHeadContextGraphId: string | null = null; private rotationHeadStalledSweeps = 0; + /** + * Where to begin the next slice's walk of one context graph's meta graphs. + * A graph whose meta graphs cannot all be measured inside one slice would + * otherwise restart at index 0 every time and yield at the same point, so + * everything past that point is never cleaned — the rotation cursor's own + * failure mode, one level down. Rotating the start covers them all across + * successive slices. + */ + private metaGraphResumeCursor: { contextGraphId: string; offset: number } | null = null; constructor(options: FinalizedSwmCleanupServiceOptions) { this.store = options.store; @@ -215,8 +224,18 @@ export class FinalizedSwmCleanupService { const contextGraphId = pending[index]!; // Yielding keeps the unvisited tail, so the cursor never advances past a // context graph this sweep did not finish measuring. + let metaGraphStart = 0; + let metaGraphsMeasured = 0; const yieldRotation = (reason: 'pressure' | 'budget'): FinalizedSwmCleanupSweepResult => { this.rotationPending = pending.slice(index); + // Resume the meta-graph walk past what this slice already measured, so + // a context graph too large for one slice still covers all of them. + if (metaGraphsMeasured > 0) { + this.metaGraphResumeCursor = { + contextGraphId, + offset: metaGraphStart + metaGraphsMeasured, + }; + } return deferredResult(deletedItems, reason); }; if (underPressure()) return yieldRotation('pressure'); @@ -229,6 +248,16 @@ export class FinalizedSwmCleanupService { if (isTransientStorePressure(error)) return yieldRotation('pressure'); throw error; } + // Resume where the last slice stopped inside this context graph, wrapping, + // so a graph too large for one slice still reaches every meta graph rather + // than re-walking the same prefix. The walk still covers all of them, so + // the contribution below stays a whole-graph sum. + if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId && metaGraphs.length > 0) { + metaGraphStart = this.metaGraphResumeCursor.offset % metaGraphs.length; + if (metaGraphStart > 0) { + metaGraphs = [...metaGraphs.slice(metaGraphStart), ...metaGraphs.slice(0, metaGraphStart)]; + } + } // A context graph contributes to the rotation total only once every one of // its meta graphs has been measured. Yielding part-way discards the partial // sum, so the re-measure on resume cannot double-count it. @@ -281,8 +310,13 @@ export class FinalizedSwmCleanupService { ) { contextGraphOldestMarkerAt = backlog.oldestMarkerAt; } + metaGraphsMeasured += 1; await new Promise((resolve) => setImmediate(resolve)); } + // Every meta graph was measured, so the next visit starts from the top. + if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId) { + this.metaGraphResumeCursor = null; + } this.rotationBacklogDepth += contextGraphBacklogDepth; if ( contextGraphOldestMarkerAt !== null From ec59670a43f129509196663f380a3a96a103ae84 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:37:41 +0200 Subject: [PATCH 39/48] fix(agent): persist the rotation cursor when a sweep faults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A terminal fault mid-rotation rethrew without touching `rotationPending`. Context graphs completed earlier in the same sweep were already credited to `rotationBacklogDepth`, and resumeRotation only zeroes the accumulator when `rotationPending` is null — which a throw never produces. The next sweep resumed from the stale tail, re-measured those graphs and added them again, and the inflated sum eventually published as `stale: false`: presented as a trustworthy fresh whole-node measurement. Reproduced on three context graphs of 10 markers each: true total 30, reported 30 -> 40 after one fault. Requires the rotation to have been resumed (a prior yield left `rotationPending` non-null); a fault during a fresh rotation is already safe because resumeRotation zeroes the accumulator on entry. Catch around the whole loop rather than at each of the four throw sites, so a raise site added later cannot miss the requirement — enumerating sites is how three of the four pressure conversions went unpinned earlier in this change set. The loop body is re-indented into the try; `git diff -w` shows the change is four lines. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 197 ++++++++++-------- 1 file changed, 106 insertions(+), 91 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index f3d942d55c..2f3eeb262e 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -220,113 +220,128 @@ export class FinalizedSwmCleanupService { } const pending = this.resumeRotation(contextGraphIds); - for (let index = 0; index < pending.length; index += 1) { - const contextGraphId = pending[index]!; - // Yielding keeps the unvisited tail, so the cursor never advances past a - // context graph this sweep did not finish measuring. - let metaGraphStart = 0; - let metaGraphsMeasured = 0; - const yieldRotation = (reason: 'pressure' | 'budget'): FinalizedSwmCleanupSweepResult => { - this.rotationPending = pending.slice(index); - // Resume the meta-graph walk past what this slice already measured, so - // a context graph too large for one slice still covers all of them. - if (metaGraphsMeasured > 0) { - this.metaGraphResumeCursor = { - contextGraphId, - offset: metaGraphStart + metaGraphsMeasured, - }; - } - return deferredResult(deletedItems, reason); - }; - if (underPressure()) return yieldRotation('pressure'); - if (this.now() >= deadline) return yieldRotation('budget'); - let metaGraphs: string[]; - try { - metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId, discoveryOptions); - } catch (error) { - if (deadlineSignal.aborted) return yieldRotation('budget'); - if (isTransientStorePressure(error)) return yieldRotation('pressure'); - throw error; - } - // Resume where the last slice stopped inside this context graph, wrapping, - // so a graph too large for one slice still reaches every meta graph rather - // than re-walking the same prefix. The walk still covers all of them, so - // the contribution below stays a whole-graph sum. - if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId && metaGraphs.length > 0) { - metaGraphStart = this.metaGraphResumeCursor.offset % metaGraphs.length; - if (metaGraphStart > 0) { - metaGraphs = [...metaGraphs.slice(metaGraphStart), ...metaGraphs.slice(0, metaGraphStart)]; - } - } - // A context graph contributes to the rotation total only once every one of - // its meta graphs has been measured. Yielding part-way discards the partial - // sum, so the re-measure on resume cannot double-count it. - let contextGraphBacklogDepth = 0; - let contextGraphOldestMarkerAt: number | null = null; - for (const swmMetaGraph of metaGraphs) { - if (underPressure()) return yieldRotation('pressure'); - if (this.now() >= deadline) return yieldRotation('budget'); - if (remaining > 0) { - let cleanup: { deletedItems: number; examinedCandidates: number }; - try { - cleanup = await this.cleanupMetaGraph({ + let index = 0; + try { + for (; index < pending.length; index += 1) { + const contextGraphId = pending[index]!; + // Yielding keeps the unvisited tail, so the cursor never advances past a + // context graph this sweep did not finish measuring. + let metaGraphStart = 0; + let metaGraphsMeasured = 0; + const yieldRotation = (reason: 'pressure' | 'budget'): FinalizedSwmCleanupSweepResult => { + this.rotationPending = pending.slice(index); + // Resume the meta-graph walk past what this slice already measured, so + // a context graph too large for one slice still covers all of them. + if (metaGraphsMeasured > 0) { + this.metaGraphResumeCursor = { contextGraphId, - swmMetaGraph, - maxCandidates: remaining, - signal: deadlineSignal, - }); - } catch (error) { - if (deadlineSignal.aborted) return yieldRotation('budget'); - if (isTransientStorePressure(error)) return yieldRotation('pressure'); - throw error; + offset: metaGraphStart + metaGraphsMeasured, + }; } - deletedItems += cleanup.deletedItems; - remaining -= cleanup.examinedCandidates; - } + return deferredResult(deletedItems, reason); + }; if (underPressure()) return yieldRotation('pressure'); if (this.now() >= deadline) return yieldRotation('budget'); - // Deliberately not gated on `remaining`: that budget bounds deletion - // work (head resolution, VM/SWM verification, lock, atomic replace), - // while this is one cheap background aggregate and the sole source of - // the backlog metric. Capping the metric with the deletion budget would - // make it under-report exactly when the backlog is deepest. - let backlog: { depth: number; oldestMarkerAt: number | null }; + let metaGraphs: string[]; try { - backlog = await this.inspectBacklog(swmMetaGraph, deadlineSignal); + metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId, discoveryOptions); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); - // Also covers this method's own defensive pressure throw, which the - // snapshot gate above races with rather than prevents. if (isTransientStorePressure(error)) return yieldRotation('pressure'); throw error; } - contextGraphBacklogDepth += backlog.depth; + // Resume where the last slice stopped inside this context graph, wrapping, + // so a graph too large for one slice still reaches every meta graph rather + // than re-walking the same prefix. The walk still covers all of them, so + // the contribution below stays a whole-graph sum. + if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId && metaGraphs.length > 0) { + metaGraphStart = this.metaGraphResumeCursor.offset % metaGraphs.length; + if (metaGraphStart > 0) { + metaGraphs = [...metaGraphs.slice(metaGraphStart), ...metaGraphs.slice(0, metaGraphStart)]; + } + } + // A context graph contributes to the rotation total only once every one of + // its meta graphs has been measured. Yielding part-way discards the partial + // sum, so the re-measure on resume cannot double-count it. + let contextGraphBacklogDepth = 0; + let contextGraphOldestMarkerAt: number | null = null; + for (const swmMetaGraph of metaGraphs) { + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + if (remaining > 0) { + let cleanup: { deletedItems: number; examinedCandidates: number }; + try { + cleanup = await this.cleanupMetaGraph({ + contextGraphId, + swmMetaGraph, + maxCandidates: remaining, + signal: deadlineSignal, + }); + } catch (error) { + if (deadlineSignal.aborted) return yieldRotation('budget'); + if (isTransientStorePressure(error)) return yieldRotation('pressure'); + throw error; + } + deletedItems += cleanup.deletedItems; + remaining -= cleanup.examinedCandidates; + } + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + // Deliberately not gated on `remaining`: that budget bounds deletion + // work (head resolution, VM/SWM verification, lock, atomic replace), + // while this is one cheap background aggregate and the sole source of + // the backlog metric. Capping the metric with the deletion budget would + // make it under-report exactly when the backlog is deepest. + let backlog: { depth: number; oldestMarkerAt: number | null }; + try { + backlog = await this.inspectBacklog(swmMetaGraph, deadlineSignal); + } catch (error) { + if (deadlineSignal.aborted) return yieldRotation('budget'); + // Also covers this method's own defensive pressure throw, which the + // snapshot gate above races with rather than prevents. + if (isTransientStorePressure(error)) return yieldRotation('pressure'); + throw error; + } + contextGraphBacklogDepth += backlog.depth; + if ( + backlog.oldestMarkerAt !== null + && ( + contextGraphOldestMarkerAt === null + || backlog.oldestMarkerAt < contextGraphOldestMarkerAt + ) + ) { + contextGraphOldestMarkerAt = backlog.oldestMarkerAt; + } + metaGraphsMeasured += 1; + await new Promise((resolve) => setImmediate(resolve)); + } + // Every meta graph was measured, so the next visit starts from the top. + if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId) { + this.metaGraphResumeCursor = null; + } + this.rotationBacklogDepth += contextGraphBacklogDepth; if ( - backlog.oldestMarkerAt !== null + contextGraphOldestMarkerAt !== null && ( - contextGraphOldestMarkerAt === null - || backlog.oldestMarkerAt < contextGraphOldestMarkerAt + this.rotationOldestMarkerAt === null + || contextGraphOldestMarkerAt < this.rotationOldestMarkerAt ) ) { - contextGraphOldestMarkerAt = backlog.oldestMarkerAt; + this.rotationOldestMarkerAt = contextGraphOldestMarkerAt; } - metaGraphsMeasured += 1; - await new Promise((resolve) => setImmediate(resolve)); - } - // Every meta graph was measured, so the next visit starts from the top. - if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId) { - this.metaGraphResumeCursor = null; - } - this.rotationBacklogDepth += contextGraphBacklogDepth; - if ( - contextGraphOldestMarkerAt !== null - && ( - this.rotationOldestMarkerAt === null - || contextGraphOldestMarkerAt < this.rotationOldestMarkerAt - ) - ) { - this.rotationOldestMarkerAt = contextGraphOldestMarkerAt; } + } catch (error) { + // Every exit from this loop must persist the cursor, including a terminal + // fault. Context graphs completed earlier in this sweep are already + // credited to `rotationBacklogDepth`, and `resumeRotation` only zeroes the + // accumulator when `rotationPending` is null — which a throw never + // produces. Leaving the cursor stale makes the next sweep resume from the + // old tail, re-measure those graphs and add them a second time, and the + // inflated sum eventually publishes as `stale: false`: presented as a + // trustworthy fresh whole-node measurement. Catching around the whole loop + // rather than at each throw site means a future raise site cannot miss it. + this.rotationPending = pending.slice(index); + throw error; } // The rotation closed: every context graph was measured exactly once, so the From 9e9f41cda4dd48f9d0ea9b82864c8e88ebca0e90 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:42:37 +0200 Subject: [PATCH 40/48] test(agent): stop the rotation tests racing a real-time deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runSweep derives two deadlines from wallClockBudgetMs: one from the injected clock, and one from AbortSignal.timeout on real wall time that is threaded into the store and aborts its queries. Only the first is controllable from a test, and these tests injected a clock while setting a 100ms budget — so any slice taking longer than 100ms of real time yielded at a boundary the injected clock never chose. Reproduced deterministically before fixing: with the injected clock FROZEN, so the injected deadline can never fire, a 150ms store query against a 100ms budget still returned budgetExhausted: true after 158ms real. The yield came exclusively from the real timer. That matters more than ordinary flakiness here, because these are the tests pinning the cursor's forward-progress guarantee — the property that turned out to be broken. A non-deterministic test on a guarantee that failed is close to no test at all, and on a loaded machine or one running several worktrees it reddens for reasons unrelated to the code. The budget now exceeds any plausible real slice duration and the tick is expressed as a fraction of it, so the injected clock alone decides every deadline yield. The header records why, since the obvious tidying is to shrink these back to small round numbers. All five rotation mutants still die, so the tests remain load-bearing rather than merely stable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../finalized-swm-cleanup-rotation.test.ts | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts index 54c05974f1..bad6c16701 100644 --- a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts +++ b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts @@ -30,6 +30,26 @@ import { } from '../src/dkg-agent-constants.js'; import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; +/** + * `runSweep` derives TWO deadlines from `wallClockBudgetMs`: one from the + * injected `now`, and one from `AbortSignal.timeout(...)` on REAL wall time. + * The second is threaded into the store and aborts its queries, and nothing in + * a test can control it. + * + * The budget must therefore exceed any plausible real slice duration. With a + * small budget, a slow machine, a loaded CI box or a neighbouring worktree + * makes the real timer fire first and the sweep yields at a boundary the + * injected clock never chose — reddening these tests for reasons unrelated to + * the code they pin. Measured directly: with the injected clock frozen and a + * 100 ms budget, a 150 ms store query still returns `budgetExhausted: true`. + * + * The injected clock alone must decide every deadline yield here, so the tick + * is expressed as a fraction of the budget: two ticks overrun one slice, one + * does not. Do not shrink these to make the arithmetic look tidier. + */ +const SLICE_BUDGET_MS = 60_000; +const SLICE_TICK_MS = Math.ceil(SLICE_BUDGET_MS * 0.6); + const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; const XSD_DATE_TIME = 'http://www.w3.org/2001/XMLSchema#dateTime'; const MARKER_ROOT = JSON.stringify(`0x${'11'.repeat(32)}`); @@ -144,10 +164,11 @@ describe('finalized SWM cleanup rotation cursor', () => { * every retry re-walked the same prefix, so markers in later context graphs * were never discovered no matter how many times the worker woke. * - * Four context graphs, 10 markers each, enumeration costing 60ms against a - * 100ms slice. Context graphs legitimately repeat across slices — a graph - * entered but not finished is re-measured whole — so the assertion is on the - * entry sequence and on the single published total, not on disjoint sets. + * Four context graphs, 10 markers each, enumeration costing one tick against + * a slice that fits fewer than two. Context graphs legitimately repeat across + * slices — a graph entered but not finished is re-measured whole — so the + * assertion is on the entry sequence and on the single published total, not + * on disjoint sets. */ it('resumes at the unvisited tail and publishes a whole-node total only when the rotation closes', async () => { const store = new OxigraphStore(); @@ -168,11 +189,11 @@ describe('finalized SWM cleanup rotation cursor', () => { store, writeLocks: new Map>(), now: () => clock.now, - wallClockBudgetMs: 100, + wallClockBudgetMs: SLICE_BUDGET_MS, listContextGraphIds: async () => [...contextGraphIds], listSharedMemoryMetaGraphs: async (contextGraphId) => { currentSweep.push(contextGraphId); - clock.now += 60; + clock.now += SLICE_TICK_MS; return [metaByContextGraph.get(contextGraphId)!]; }, }); @@ -233,12 +254,12 @@ describe('finalized SWM cleanup rotation cursor', () => { store, writeLocks: new Map>(), now: () => clock.now, - wallClockBudgetMs: 100, + wallClockBudgetMs: SLICE_BUDGET_MS, listContextGraphIds: async () => ['cg-slow', 'cg-b', 'cg-c'], listSharedMemoryMetaGraphs: async (contextGraphId) => { currentSweep.push(contextGraphId); // The head alone costs more than a whole slice, so it can never finish. - clock.now += contextGraphId === 'cg-slow' ? 200 : 10; + clock.now += contextGraphId === 'cg-slow' ? SLICE_BUDGET_MS * 2 : Math.ceil(SLICE_BUDGET_MS * 0.1); if (contextGraphId === 'cg-slow') return [slowMeta]; return [contextGraphId === 'cg-b' ? metaB : metaC]; }, @@ -323,10 +344,10 @@ describe('finalized SWM cleanup rotation cursor', () => { store, writeLocks: new Map>(), now: () => clock.now, - wallClockBudgetMs: 100, + wallClockBudgetMs: SLICE_BUDGET_MS, listContextGraphIds: async () => [...live], listSharedMemoryMetaGraphs: async (contextGraphId) => { - clock.now += 60; + clock.now += SLICE_TICK_MS; return [contextGraphId === 'cg-a' ? metaA : metaB]; }, }); From 650174547f61a5487b688277a05039ee9f406dd5 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:53:25 +0200 Subject: [PATCH 41/48] fix(agent): page task discovery with a keyset cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery was ORDER BY ?task LIMIT n with no cursor, so every sweep re-selected the same prefix. A task that can never be acted on — a marker armed from VM state whose live SWM diverges, which finalization-handler reaches whenever VM verifies — returns `preserved` on every sweep and is charged to the node-wide deletion budget every time. Four such rows at the ORDER BY head are enough that no other meta graph gets a candidate query again. The charging is not the bug and is deliberately unchanged: examination is the expensive operation (a head resolve, plus two verifyExactGraphScopedLayer passes per matching row), so counting only deletions would uncap exactly the work the budget bounds. Selection is the bug. Carry one keyset position, advancing while the page is full and clearing when it is short — without the wrap this trades a stuck prefix for a permanently stranded suffix, the same starvation with the opposite sign. A single entry is bounded by construction rather than by an eviction policy: charging per examined candidate means a full page spends the rest of the budget, so at most one meta graph can hold an unfinished page. Only runSweep supplies the cursor, so the cleanupKnownMetaGraph seam and its ~dozen drainFinalizedSwm callers keep selecting from the top unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../src/finalized-swm-cleanup-service.ts | 60 +++++++++++++++++-- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index 2f3eeb262e..061e882019 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -148,6 +148,22 @@ export class FinalizedSwmCleanupService { * successive slices. */ private metaGraphResumeCursor: { contextGraphId: string; offset: number } | null = null; + /** + * Keyset position within the one meta graph whose candidate page is unfinished. + * Discovery is `ORDER BY ?task LIMIT n`, so without this the same prefix is + * re-selected every sweep and its successors are never examined — the third + * instance of the cursor-less re-walk this service has, after the context-graph + * rotation and the meta-graph walk. Tasks that can never be acted on (a head + * that no longer verifies) otherwise hold the whole node-wide deletion budget + * indefinitely. + * + * A single entry suffices, and is bounded by construction rather than by an + * eviction policy: charging `remaining` per examined candidate means a full page + * always spends the rest of the budget, so no other meta graph runs discovery + * in that slice, and a short page clears the cursor. At most one meta graph can + * hold an unfinished page at a time. + */ + private taskCursor: { swmMetaGraph: string; afterTaskSubject: string } | null = null; constructor(options: FinalizedSwmCleanupServiceOptions) { this.store = options.store; @@ -269,13 +285,16 @@ export class FinalizedSwmCleanupService { if (underPressure()) return yieldRotation('pressure'); if (this.now() >= deadline) return yieldRotation('budget'); if (remaining > 0) { - let cleanup: { deletedItems: number; examinedCandidates: number }; + let cleanup: Awaited>; try { cleanup = await this.cleanupMetaGraph({ contextGraphId, swmMetaGraph, maxCandidates: remaining, signal: deadlineSignal, + ...(this.taskCursor?.swmMetaGraph === swmMetaGraph + ? { afterTaskSubject: this.taskCursor.afterTaskSubject } + : {}), }); } catch (error) { if (deadlineSignal.aborted) return yieldRotation('budget'); @@ -284,6 +303,11 @@ export class FinalizedSwmCleanupService { } deletedItems += cleanup.deletedItems; remaining -= cleanup.examinedCandidates; + // Advance while the page is full; wrap when it is short so a stuck + // prefix is skipped without stranding the suffix behind it. + this.taskCursor = cleanup.pageWasFull && cleanup.lastTaskSubject !== undefined + ? { swmMetaGraph, afterTaskSubject: cleanup.lastTaskSubject } + : null; } if (underPressure()) return yieldRotation('pressure'); if (this.now() >= deadline) return yieldRotation('budget'); @@ -487,9 +511,19 @@ export class FinalizedSwmCleanupService { swmMetaGraph: string; maxCandidates?: number; signal?: AbortSignal; - }): Promise<{ deletedItems: number; examinedCandidates: number }> { + /** + * Keyset position from a previous slice. Only `runSweep` supplies it, so the + * `cleanupKnownMetaGraph` seam keeps selecting from the top unchanged. + */ + afterTaskSubject?: string; + }): Promise<{ + deletedItems: number; + examinedCandidates: number; + lastTaskSubject?: string; + pageWasFull: boolean; + }> { if (!this.writeLocks || hasActiveStorePressure(this.store.getPressureSnapshot?.())) { - return { deletedItems: 0, examinedCandidates: 0 }; + return { deletedItems: 0, examinedCandidates: 0, pageWasFull: false }; } const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); const queryOptions: QueryOptions = { @@ -508,11 +542,16 @@ export class FinalizedSwmCleanupService { ?shareId ; ?assertionGraph . OPTIONAL { ?task ?subGraphName } + ${input.afterTaskSubject === undefined + ? '' + : `FILTER(STR(?task) > ${JSON.stringify(input.afterTaskSubject)})`} } } ORDER BY ?task LIMIT ${limit}`, queryOptions, ); - if (result.type !== 'bindings') return { deletedItems: 0, examinedCandidates: 0 }; + if (result.type !== 'bindings') { + return { deletedItems: 0, examinedCandidates: 0, pageWasFull: false }; + } let cleared = 0; for (const row of result.bindings) { @@ -601,7 +640,18 @@ export class FinalizedSwmCleanupService { }); if (outcome === 'cleared') cleared += 1; } - return { deletedItems: cleared, examinedCandidates: result.bindings.length }; + // `ORDER BY ?task` makes the last row the keyset position. A short page means + // the ordered set is exhausted, so the caller wraps instead of advancing — + // without that, skipping a stuck prefix would trade it for a permanently + // skipped suffix, the same starvation with the opposite sign. + const lastRow = result.bindings[result.bindings.length - 1]; + const lastTaskSubject = typeof lastRow?.['task'] === 'string' ? lastRow['task'] : undefined; + return { + deletedItems: cleared, + examinedCandidates: result.bindings.length, + ...(lastTaskSubject === undefined ? {} : { lastTaskSubject }), + pageWasFull: result.bindings.length >= limit, + }; } private async retireStaleTask(input: { From cc0fa0704e7256b743c0ee4521d4b89f23e06b28 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:55:39 +0200 Subject: [PATCH 42/48] docs(agent): correct the short-page wrap claim in 650174547 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 650174547 states that without the short-page wrap a stuck prefix is traded for a permanently stranded suffix. Mutation testing refutes that: a mutant that advances the cursor on short pages too is NOT killed, because an empty page carries no last subject and clears the cursor anyway. Dropping the wrap costs one wasted empty query per cycle; it does not strand anything. The wrap is kept — one query per cycle is worth the line — but described as the optimisation it is. I asserted the stronger claim in a commit message and a code comment without testing the negative, which is the same failure this change set has been correcting elsewhere. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/finalized-swm-cleanup-service.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts index 061e882019..21597cfc01 100644 --- a/packages/agent/src/finalized-swm-cleanup-service.ts +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -641,9 +641,12 @@ export class FinalizedSwmCleanupService { if (outcome === 'cleared') cleared += 1; } // `ORDER BY ?task` makes the last row the keyset position. A short page means - // the ordered set is exhausted, so the caller wraps instead of advancing — - // without that, skipping a stuck prefix would trade it for a permanently - // skipped suffix, the same starvation with the opposite sign. + // the ordered set is exhausted, so the caller wraps instead of advancing. + // This is an optimisation, NOT a safety property: an empty page carries no + // last subject and therefore clears the cursor anyway, so dropping the + // short-page wrap costs one wasted empty query per cycle rather than + // stranding the suffix. Established by mutation — a wrap-only mutant is not + // killed, and this fallback is why. const lastRow = result.bindings[result.bindings.length - 1]; const lastTaskSubject = typeof lastRow?.['task'] === 'string' ? lastRow['task'] : undefined; return { From 5cdd805c9c50edcfced8b9a26f37d5b4a8a9d211 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 14:01:28 +0200 Subject: [PATCH 43/48] test(agent): record a cursor mutant that new call sites cannot outgrow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep loop keeps growing rotation-cursor persist sites — a fault path, then a task-selection cursor — and a mutant that enumerates them weakens silently every time one is added. The enumeration is complete when written, the code grows underneath it, and nothing announces that the mutant now neutralises only part of what it names. That is harder to notice than ordinary under-enumeration, because there is no point at which anyone did anything wrong. Records the site-independent form instead: neutralise the cursor at its single declaration with a getter/setter pair that discards writes. It cannot be outgrown by new assignment sites, so it stays valid across restructures of the loop rather than needing maintenance alongside them. Verified against the current shape, which already has two persist sites plus a clear: the declaration mutant kills the resume test on its own. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/finalized-swm-cleanup-rotation.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts index bad6c16701..d9bc075a8d 100644 --- a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts +++ b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts @@ -14,6 +14,20 @@ * context graphs each slice actually entered. Asserting only that the cursor * advanced is much weaker: it passes with the accumulator committed in the * wrong place, which is the mutation that silently inflates the SLO surface. + * + * VERIFYING THIS SUITE STILL BITES. Neutralise the cursor at its DECLARATION, + * not at its assignment sites: + * + * private rotationPendingDiscarded: string[] | null = null; + * private get rotationPending(): string[] | null { return null; } + * private set rotationPending(_v: string[] | null) {} + * + * The sweep loop has repeatedly grown new persist sites — a fault path, then a + * task cursor — and a mutant that enumerates them silently weakens each time + * one is added: the enumeration was complete when written, the code grew + * underneath it, and nothing fails to announce that. Mutating the single + * declaration cannot be outgrown by new call sites, so it stays valid across + * restructures of the loop. */ import { describe, expect, it, vi } from 'vitest'; From 7bb0ac3a4042edae475e59c58f7cc3daa09edb31 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 14:04:52 +0200 Subject: [PATCH 44/48] test(agent): separate site-proof from blanket mutants in the note The header now tells the next person to neutralise the rotation cursor at its declaration, and that instruction is easy to over-apply: "one mutation covering everything" is exactly the failure this change set spent its time removing, where four conversion sites were killed by one test and three predicates were asserted in one case. The two are not the same. A blanket mutant stands in for many properties, so its kill says something broke without saying what. A site-proof mutant covers one property and is merely robust to that property gaining implementation sites. Records the check that separates them: whether the kill set stays narrow and specific. The declaration mutation kills the resume test and nothing else. A site-proof mutant that starts reddening half the suite has become blanket, and that is a signal to split it rather than a stronger result. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../test/finalized-swm-cleanup-rotation.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts index d9bc075a8d..3550053d07 100644 --- a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts +++ b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts @@ -28,6 +28,20 @@ * underneath it, and nothing fails to announce that. Mutating the single * declaration cannot be outgrown by new call sites, so it stays valid across * restructures of the loop. + * + * This is NOT the same as a blanket mutant, and the difference decides whether + * a result means anything: + * + * - A BLANKET mutant stands in for many PROPERTIES. That is the failure to + * avoid — four conversion sites killed by one test, three predicates + * asserted in one case. Its kill tells you something broke, not what. + * - A SITE-PROOF mutant covers ONE property and is merely robust to that + * property gaining implementation sites. + * + * The check is whether the kill set stays narrow and specific: the mutation + * above kills the resume test and nothing else. If a site-proof mutant starts + * reddening half the suite it has become blanket, and that is the signal to + * split it rather than to celebrate the kill. */ import { describe, expect, it, vi } from 'vitest'; From 6ea5a38e93380b1a796b5cea06524dc7def2c6da Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 13:43:51 +0200 Subject: [PATCH 45/48] fix(agent): serialize the cleanup-marker write with the per-KA SWM lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit markFinalizedGraphScopedSwmForCleanup wrote the tombstone with a bare store.insert while every other writer of that operation subject takes swmKaWriteLockKey. Catch-up REPLACES the same subject under that lock: it reads the tombstone, deletes the subject, then re-inserts from the snapshot. An unlocked marker lands inside that window — it runs at default priority while the replace's reads and deletes are queued at background, so it is admitted ahead of them — and the re-insert restores a snapshot taken before it existed. The tombstone is then gone while the independent task subject survives, so the GC cleans the lifecycle once and retires the task, and the next catch-up re-materializes the SWM copy with nothing left to re-arm cleanup from. That is a permanent resurrection of exactly what this PR removes. Taking the lock is compatible with keeping finalization foreground work O(1) because that constraint is on WORK, not on waiting. This path already reads and hashes the entire SWM payload via verifyExactGraphScopedLayer before reaching the marker write, so a bounded wait behind one KA's replace is smaller than what it has already done, and it adds no cleanup discovery, verification, deletion, or GC wait. The insert is a leaf operation, so the lock cannot nest and cannot deadlock. writeLocks returns to FinalizationHandler injected and READ. The field deleted earlier in this branch was dead — assigned, never used — which is worse than absent because it advertised a serialization that never happened, and is why this race survived review. Rejected alternatives: re-reading the tombstone later inside the lock only narrows the window, and replaceSubjectAtomicallyOrFallback does not close it either, since the payload is still snapshotted before the atomic boundary. Moving the tombstone to a subject catch-up never deletes is the better end state but is a persisted-shape change with an upgrade path, filed separately. The test injects the interleaving rather than racing for it: the marker is started as catch-up is about to delete the operation subject and given twenty event-loop turns. Unlocked it completes every time; locked it cannot complete at all, because the replace holds the lock — so the assertion turns on mutual exclusion, not timing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/dkg-agent-swm-substrate.ts | 3 + packages/agent/src/finalization-handler.ts | 45 ++++++++- .../test/swm-snapshot-materializer.test.ts | 95 ++++++++++++++++++- 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 90e5427dc5..dc84d9e25e 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -1654,6 +1654,9 @@ export class SwmSubstrateMethods extends DKGAgentBase { markContextGraphMetaDirtyFromQuads: (quads) => { this.contextGraphMetaProjection.markDirtyFromQuads(quads); }, + // The SAME map catch-up and the finalized-SWM GC take, so the cleanup + // marker write serializes against them on the per-KA key. + writeLocks: this.writeLocks, publicSnapshotStore: this.publicSnapshotStore, wakeFinalizedSwmCleanup: () => this.wakeFinalizedSwmCleanup(), runtime: this.finalizationRuntime, diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index ae6806f6f5..851810ff14 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -49,6 +49,8 @@ import { KnowledgeAssetWorkspaceHeadCorruptError, resolveKnowledgeAssetOperationPublicQuads, resolveKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, workspaceOperationSubject, workspacePublicQuadsDigest, type MaterializedVersion, @@ -292,6 +294,13 @@ export interface FinalizationHandlerOptions { eventBus?: EventBus; resolveContextGraphOnChainId?: ResolveContextGraphOnChainId; markContextGraphMetaDirtyFromQuads?: MarkContextGraphMetaDirtyFromQuads; + /** + * The SAME per-KA SWM writer lock map catch-up and the finalized-SWM GC use. + * Taken around the cleanup-marker write so it cannot land inside catch-up's + * read-delete-reinsert of the operation subject. Absent => that write is + * unserialized; see `markFinalizedGraphScopedSwmForCleanup`. + */ + writeLocks?: Map>; publicSnapshotStore?: WorkspacePublicSnapshotStore; wakeFinalizedSwmCleanup?: () => void; lifecycleLogOptions?: FinalizationLifecycleLogOptions; @@ -363,6 +372,7 @@ export class FinalizationHandler { private readonly eventBus: EventBus | undefined; private readonly resolveContextGraphOnChainId: ResolveContextGraphOnChainId | undefined; private readonly markContextGraphMetaDirtyFromQuads: MarkContextGraphMetaDirtyFromQuads | undefined; + private readonly writeLocks: Map> | undefined; private readonly publicSnapshotStore: WorkspacePublicSnapshotStore | undefined; private readonly wakeFinalizedSwmCleanup: (() => void) | undefined; private readonly recovery: FinalizationRecovery; @@ -424,6 +434,7 @@ export class FinalizationHandler { this.eventBus = options.eventBus; this.resolveContextGraphOnChainId = options.resolveContextGraphOnChainId; this.markContextGraphMetaDirtyFromQuads = options.markContextGraphMetaDirtyFromQuads; + this.writeLocks = options.writeLocks; this.publicSnapshotStore = options.publicSnapshotStore; this.wakeFinalizedSwmCleanup = options.wakeFinalizedSwmCleanup; this.lifecycle = new FinalizationLifecycleLogger( @@ -1589,7 +1600,7 @@ export class FinalizationHandler { const cleanupRoot = JSON.stringify(cleanupRootHex); const markedAtIso = new Date().toISOString(); const markedAt = `"${markedAtIso}"^^`; - await this.store.insert([ + const writeMarker = () => this.store.insert([ ...buildFinalizedSwmCleanupTaskQuads({ contextGraphId: input.contextGraphId, subGraphName: input.subGraphName, @@ -1611,6 +1622,38 @@ export class FinalizationHandler { graph: metaGraph, }, ]); + // Serialize against the other writers of this operation subject. The + // tombstone lands on `operationSubject`, which catch-up REPLACES under this + // same lock: it reads the tombstone, then deletes the subject, then + // re-inserts from that snapshot. An unlocked marker write can land inside + // that window — the marker runs at default priority while the replace's + // reads and deletes are queued at background, so it is admitted ahead of + // them — and the re-insert then restores a snapshot taken before the marker + // existed. The tombstone is gone, the independent task survives, the GC + // cleans the lifecycle once and retires the task, and the next catch-up + // re-materializes the SWM copy with nothing left to re-arm cleanup. That is + // a PERMANENT resurrection of exactly what this component exists to remove. + // + // This is compatible with keeping finalization foreground work O(1): the + // constraint is on WORK, not on waiting. This path already read and hashed + // the entire SWM payload via `verifyExactGraphScopedLayer` before reaching + // here, so a bounded wait behind one KA's replace is smaller than what it + // has already done, and it adds no cleanup discovery, verification, + // deletion, or GC wait. The insert is a leaf operation, so taking the lock + // here cannot nest and cannot deadlock. + if (this.writeLocks) { + await withKeyedLocks( + this.writeLocks, + [swmKaWriteLockKey(input.contextGraphId, input.subGraphName, input.scope.ual)], + writeMarker, + ); + } else { + // No shared lock map wired (test/embedded construction). Still mark — + // refusing would leave the finalized copy with no cleanup record at all, + // which is worse than an unserialized write — but this path is NOT + // protected against the interleaving above. + await writeMarker(); + } this.wakeFinalizedSwmCleanup?.(); return 'marked'; } diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index 46c8caac1f..f934825e46 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -33,15 +33,22 @@ import { import { generateKnowledgeAssetShareMetadata, resolveKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, workspacePublicQuadsDigest, + type KnowledgeAssetWorkspaceHead, type WorkspacePublicSnapshotStore, } from '@origintrail-official/dkg-publisher'; import { GraphManager, OxigraphStore, type Quad, type TripleStore } from '@origintrail-official/dkg-storage'; import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped-swm-recovery.js'; -import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; +import { + createSharedMemorySnapshotMaterializer, + replaceGraphScopedSwmHeadMetadata, +} from '../src/sync/requester/swm-snapshot-materializer.js'; import { runSharedMemorySync } from '../src/sync/requester/shared-memory-sync.js'; import type { SyncPageResult } from '../src/sync/requester/page-fetch.js'; import { + FinalizationHandler, FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, FINALIZED_SWM_CLEANUP_TASK_TYPE, @@ -629,4 +636,90 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', } }); }); + + /** + * The cleanup-marker write must serialize against catch-up's replacement of + * the SAME operation subject. + * + * `replaceGraphScopedSwmHeadMetadata` reads the tombstone, deletes the + * operation subject, then re-inserts from that snapshot. A marker written + * inside that window is destroyed by the delete and never restored, because + * the snapshot predates it. The independent task subject survives, so the GC + * cleans the lifecycle once and retires the task — and the next catch-up + * re-materializes the SWM copy with no tombstone left to re-arm cleanup from. + * The finalized copy is then resurrected permanently, which is the exact + * failure this component exists to prevent. + * + * The interleaving is injected rather than raced: a marker write that only + * *sometimes* lands in the window is not a proof, and this is precisely the + * race that passes ninety-nine runs in a hundred. The marker is started at + * the moment catch-up is about to delete the operation subject and given + * twenty event-loop turns to complete. Unlocked it finishes inside that + * window every time; correctly locked it CANNOT finish, because the lock is + * held by the replace itself — so the assertion turns on mutual exclusion, + * not on timing. + */ + it('preserves the tombstone when a cleanup marker lands during a head replace', async () => { + const store = new OxigraphStore(); + await store.insert([...v1.meta]); + const writeLocks = new Map>(); + const handler = new FinalizationHandler(store, undefined as never, { writeLocks }); + const scope = createGraphKnowledgeAssetScope(UAL, 1); + const expectedHead = { + kaUal: UAL, + assertionVersion: '1', + assertionGraph: v1.assertionGraph, + publicQuadsDigest: v1.digest, + publicTripleCount: v1.payload.length, + privateTripleCount: 0, + shareOperationId: v1.operationId, + publisherPeerId: 'peer-source', + allowedPeers: [], + } as unknown as KnowledgeAssetWorkspaceHead; + + let markerStarted = false; + let marker: Promise | undefined; + const realDeleteByPattern = store.deleteByPattern.bind(store); + store.deleteByPattern = (async (pattern: Partial, options?: unknown) => { + if (!markerStarted && pattern.subject === v1.operationSubject) { + markerStarted = true; + marker = (handler as unknown as { + markFinalizedGraphScopedSwmForCleanup(input: unknown): Promise; + }).markFinalizedGraphScopedSwmForCleanup({ + contextGraphId: CG, + scope, + expectedHead, + expectedMerkleRoot: new Uint8Array(32).fill(0xab), + ctx, + }); + for (let turn = 0; turn < 20; turn += 1) { + await new Promise((resolve) => { setImmediate(resolve); }); + } + } + return realDeleteByPattern(pattern, options as never); + }) as typeof store.deleteByPattern; + + await withKeyedLocks(writeLocks, [swmKaWriteLockKey(CG, undefined, UAL)], () => + replaceGraphScopedSwmHeadMetadata({ + store, + contextGraphId: CG, + descriptor: descriptorFor(v1), + sourcePrefix: 'test.replaceHead', + insertReplacementMetadata: (quads) => store.insert([...quads]), + })); + await marker; + + expect(markerStarted).toBe(true); + // The tombstone the GC re-arms from must have survived the replace. + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toHaveLength(1); + // And the lifecycle itself is still intact, so this is not passing because + // the replace silently did nothing. + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v1.operationId}"`]); + }); }); From cbc8d9004add6f7b0ce684cd42c2a3034514e807 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 16:15:30 +0200 Subject: [PATCH 46/48] test(agent): pin the stale-task retirement lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retireStaleTask is the only path that removes a cleanup task whose head has moved on, and nothing reached it — not the function, not its call site. Its guards were therefore free to be removed without any test noticing. The load-bearing case is the negative one: an absent head with the payload still present must NOT retire. That is not a finished lifecycle, it is a resurrected SWM copy whose head has not been rebuilt, and retiring its task strands that copy with nothing left to collect it — the resurrection this change set exists to prevent, arriving through the retirement path rather than the deletion path. Removing that guard ships green today. The superseded case asserts the newer assertion survives intact, not merely that the stale task went: a change that retires v1 and damages v2 would pass a task-count assertion. Retirement is also pinned as inert when the store cannot report write generations, which is production behaviour on a backend that ships no tracker. Pinning the inertness is deliberate — a later decision that retiring on non-write-gen evidence is safe should have to turn this red rather than pass silently. Task presence is asserted directly because retirement is not reflected in the drain's return value, which counts reclaimed lifecycles only. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../ka-graph-finalization-handler.test.ts | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 6bdf2f3608..b982b37314 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2546,6 +2546,136 @@ describe('graph-scoped finalization handler', () => { })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); }); + /** + * `retireStaleTask` is the ONLY path that removes a cleanup task whose head + * has moved on, and no test reached it — including its call site. Its + * lifecycle transitions are asserted here. + * + * Retirement is deliberately NOT observable through the drain's return value: + * that counts reclaimed SWM lifecycles only, so a retiring sweep reports 0. + * These assert task presence directly for that reason. + */ + async function countCleanupTasks(subGraphName?: string): Promise { + const result = await store.query( + `SELECT (COUNT(DISTINCT ?task) AS ?count) WHERE { GRAPH ` + + `<${graphManager.sharedMemoryMetaUri(CG, subGraphName)}> { ?task ` + + ` ` + + `<${FINALIZED_SWM_CLEANUP_TASK_TYPE}> } }`, + ); + if (result.type !== 'bindings') throw new Error('expected cleanup task count bindings'); + const raw = result.bindings[0]?.['count'] ?? '0'; + return Number.parseInt(raw.replace(/^"|"\^\^.*$/g, ''), 10); + } + + async function deleteWorkspaceHeadSubject(): Promise { + await store.deleteByPattern({ + graph: graphManager.sharedMemoryMetaUri(CG), + subject: `${UAL}#dkg-swm-head`, + }); + } + + it('retires a superseded task without disturbing the newer assertion', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // The head moves to v2 while the task still names v1. + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + + await drainFinalizedSwm(); + + expect(await countCleanupTasks()).toBe(0); + // Asserting only that the stale task went is half a test: a change that + // retires v1 AND damages v2 would pass it. The newer lifecycle must survive + // intact, which is what makes this a lifecycle assertion. + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '2' }); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(store.query( + `ASK { GRAPH <${swmGraph}> { ` + + ` "newer" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + }); + + it('retires a headless task once its payload is gone', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + await deleteWorkspaceHeadSubject(); + await store.dropGraph(swmGraph); + + await drainFinalizedSwm(); + + expect(await countCleanupTasks()).toBe(0); + }); + + /** + * The guard this whole suite exists for. An absent head with the payload + * still present is NOT a finished lifecycle — it is a resurrected SWM copy + * whose head has not been rebuilt yet. Retiring its task strands that copy + * with nothing left to collect it, which is the resurrection #1996 exists to + * prevent, reached through the retirement path instead of the deletion path. + */ + it('does not retire a headless task while its payload is still present', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // Head gone, payload deliberately left behind. + await deleteWorkspaceHeadSubject(); + expect(await store.countQuads(swmGraph)).toBe(2); + + await drainFinalizedSwm(); + + expect(await countCleanupTasks()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(2); + }); + + /** + * Retirement of a headless task is silently inert when the store cannot + * report write generations — `asGraphWriteGenSource` returns null and the + * absent-head branch returns before its payload check. Blazegraph ships no + * tracker, so this is production behaviour there, and it is indistinguishable + * from "nothing needed retiring" without a test saying otherwise. + * + * Pinning the inertness itself is deliberate: if a later change decides + * retiring on non-write-gen evidence is safe, this goes red and forces that + * to be a considered decision rather than a silent one. + */ + it('leaves a headless task untouched when the store cannot track write generations', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // Exactly the shape that retires in the test above. + await deleteWorkspaceHeadSubject(); + await store.dropGraph(swmGraph); + + const untrackedStore = withoutWriteGenTracking(store); + expect(asGraphWriteGenSource(untrackedStore)).toBeNull(); + await new FinalizedSwmCleanupService({ + store: untrackedStore, + writeLocks, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [graphManager.sharedMemoryMetaUri(CG)], + }).cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + }); + + expect(await countCleanupTasks()).toBe(1); + }); + it('defers durable cleanup while the store is busy and resumes after restart when idle', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); From c4c568bc7d236d383aa73d97d30ee4b97eb3697c Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 16:24:22 +0200 Subject: [PATCH 47/48] test(agent): pin the retirement path's in-lock head re-check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retireStaleTask re-resolves the head under the writer lock and declines to retire when the lifecycle it names has become current again. Nothing reached that branch: the four retirement cases all leave the head mismatched throughout, so the guard could be removed with every one of them still green. It is the twin of the re-check in clearIfStillExact, on the other destructive path. Narrower — this removes a marker rather than payload, so the damage is a finalized SWM copy left with nothing to collect it rather than data destroyed — but the same class, and reachable only by changing the head BETWEEN the two reads, which no static fixture produces. The seam is positional rather than by source: both head resolutions on this path carry the same discover source, so the fixture counts occurrences. That is recorded in the test, because inserting another query with that source ahead of these would retarget the seam silently and the test would keep passing while no longer exercising the guard. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- .../ka-graph-finalization-handler.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index b982b37314..ad85b5e183 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -2676,6 +2676,75 @@ describe('graph-scoped finalization handler', () => { expect(await countCleanupTasks()).toBe(1); }); + /** + * `retireStaleTask`'s own in-lock TOCTOU re-check — the twin of the guard in + * `clearIfStillExact`. The pre-lock triage found a stale head, but by the + * time the writer lock is held the head has moved BACK to matching, so the + * task is live again and must not be retired. Retiring it strands a finalized + * SWM copy with nothing left to collect it. + * + * Narrower than the deletion-path twin — this removes a marker rather than + * payload, so the damage is a stranded copy rather than destroyed data — but + * the same class, and unpinned for the same reason: reaching it needs the + * head to change BETWEEN the two reads, which no static fixture produces. + * + * THE SEAM IS POSITIONAL, NOT BY SOURCE. Both head resolutions on this path + * carry `source: 'agent.finalizedSwmCleanup.discover'`, so the fixture counts + * occurrences: 1st = the pre-lock triage in `cleanupMetaGraph`, 2nd = the + * in-lock re-read. Adding a third query with that source ahead of these + * silently retargets the seam — the test would keep passing while no longer + * exercising the guard. + */ + it('does not retire a task whose head becomes current again inside the lock', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // Pre-lock triage must see a stale head, so advance it to v2. + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + + const originalQuery = store.query.bind(store); + let headReads = 0; + let restoredInsideLock = false; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (query.startsWith(WORKSPACE_HEAD_READ_PREFIX)) { + headReads += 1; + if (headReads === 2 && !restoredInsideLock) { + restoredInsideLock = true; + // Drop the seam first so the restore and the rest of the commit path + // run against the real store. + querySpy.mockRestore(); + // The lifecycle this task names is current again. + await storeKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + shareOperationId: SHARE_ID, + kaUal: UAL, + assertionVersion: '1', + }); + } + } + return originalQuery(query, options); + }); + + await drainFinalizedSwm(); + + expect(restoredInsideLock).toBe(true); + // Still armed: the task describes a lifecycle that is live again. + expect(await countCleanupTasks()).toBe(1); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '1', shareOperationId: SHARE_ID }); + }); + it('defers durable cleanup while the store is busy and resumes after restart when idle', async () => { const { message, swmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); From bb2efd5a95ae694c609ea7ff1671cc88809bec0b Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Sun, 2 Aug 2026 17:21:16 +0200 Subject: [PATCH 48/48] fix(agent): filter immutable-snapshot discovery before LIMIT, not after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verifyImmutableGraphScopedSnapshot bound every same-version operation and took ORDER BY ?shareId LIMIT 16, then applied the triple-count and digest tests in JS to whatever survived. So LIMIT truncated the CANDIDATE SET, not the work: sixteen same-version operations with the wrong count filled the window and the matching snapshot was never examined. That is not a flake that clears on retry. The ordering is deterministic over stable store state, so every attempt re-derives the identical sixteen and misses the same snapshot, and chain reconciliation can never restore the VM. Reaching it needs only a KA re-shared repeatedly at one version. This path is also new in this branch and load-bearing: the fallback fires whenever the live SWM layer no longer verifies, which after the idle GC has run is the normal case for a late receipt. The branch both created the path and made it the common one. Push the discriminating tests into the WHERE clause so LIMIT truncates among MATCHING candidates. With a digest supplied the match count is ~1 and the limit stops mattering; without one, count-filtering alone collapses the field. ?count is compared numerically first so a non-canonical typed literal still matches, with the lexical form as a fallback for an untyped one. The filter must never be stricter than the JS check it precedes — one that rejects a candidate the caller would have accepted is the same missed-discovery bug wearing different clothes. The content-digest rejection memo is unchanged. It bounds repeated payload verification, which is work; this was always about the search. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HxoLY1KYH1cRrsqStKCsaH --- packages/agent/src/finalization-handler.ts | 24 ++++++- .../test/swm-snapshot-materializer.test.ts | 71 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index 851810ff14..0001986f62 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -1410,6 +1410,28 @@ export class FinalizationHandler { input.contextGraphId, input.subGraphName, ); + // Every DISCRIMINATING test must run inside the query, ahead of LIMIT. + // With only the version filter here, the limit truncated the CANDIDATE + // SET rather than the work: sixteen same-version operations with the + // wrong triple count filled the window, the matching one was never + // returned, and since `ORDER BY ?shareId` is deterministic over stable + // store state, every retry re-derived the identical sixteen and missed + // the same snapshot permanently. A KA re-shared repeatedly at one version + // reaches that without anything unusual happening. + // + // `?count` is compared numerically first, so a non-canonical typed + // literal ("02"^^xsd:integer) still matches, with the lexical form as a + // fallback for an untyped one. The JS check below parses either, and this + // filter must never be STRICTER than it — a filter that rejects a + // candidate the caller would have accepted is the same missed-discovery + // bug wearing different clothes. + const countFilter = Number.isSafeInteger(input.publicTripleCount) + ? `\n FILTER(?count = ${input.publicTripleCount}` + + ` || STR(?count) = ${JSON.stringify(String(input.publicTripleCount))})` + : ''; + const digestFilter = input.expectedPublicQuadsDigest === undefined + ? '' + : `\n FILTER(STR(?digest) = ${JSON.stringify(input.expectedPublicQuadsDigest)})`; const result = await this.store.query( `SELECT DISTINCT ?shareId ?digest ?count WHERE { GRAPH <${assertSafeIri(metaGraph)}> { @@ -1419,7 +1441,7 @@ export class FinalizationHandler { <${DKG_NS}shareOperationId> ?shareId ; <${DKG_NS}publicQuadsDigest> ?digest ; <${DKG_NS}publicQuadsCount> ?count . - FILTER(STR(?version) = ${JSON.stringify(input.scope.assertionVersion)}) + FILTER(STR(?version) = ${JSON.stringify(input.scope.assertionVersion)})${countFilter}${digestFilter} } } ORDER BY ?shareId LIMIT 16`, { source: 'agent.finalization.verifyImmutableSnapshot' }, diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index f934825e46..39de9430fa 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -31,6 +31,7 @@ import { type OperationContext, } from '@origintrail-official/dkg-core'; import { + computeFlatKCRootV10, generateKnowledgeAssetShareMetadata, resolveKnowledgeAssetWorkspaceHead, swmKaWriteLockKey, @@ -722,4 +723,74 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) .toEqual([`"${v1.operationId}"`]); }); + + /** + * Immutable-snapshot discovery must not let `LIMIT` truncate the CANDIDATE + * SET — only the work. + * + * The discovery query binds every same-version operation and takes + * `ORDER BY ?shareId LIMIT 16`. While the triple-count and digest tests ran + * in JS on the returned rows, sixteen non-matching operations sorting ahead + * of the real one filled the window and the matching snapshot was never + * examined. The ordering is deterministic over stable store state, so this is + * not a flake that clears on retry: every attempt re-derives the identical + * sixteen and strands the same recovery permanently. + * + * This lives with the materializer suite rather than the finalization suite + * on purpose: it is driven directly against the private method with a real + * store, needs no chain fixture, and keeps this branch out of a test file + * another branch is editing. + */ + it('finds a matching snapshot that sorts past the discovery limit', async () => { + const store = new OxigraphStore(); + const snapshotStore = new MemorySnapshotStore(); + const version = 4; + const scope = createGraphKnowledgeAssetScope(UAL, version); + const payload: Quad[] = [ + { subject: 'urn:late:a', predicate: 'http://schema.org/status', object: '"late-match"', graph: '' }, + { subject: 'urn:late:b', predicate: 'http://schema.org/status', object: '"late-match"', graph: '' }, + ]; + const digest = workspacePublicQuadsDigest(payload); + await snapshotStore.putSnapshot({ digest, quads: payload }); + + const operationRows = (shareId: string, publicCount: number, quadsDigest: string): Quad[] => { + const subject = `urn:dkg:share:${CG}:${shareId}`; + return [ + { subject, predicate: `${DKG}contentScopeVersion`, object: `"2"^^<${XSD_INTEGER}>`, graph: WS_META }, + { subject, predicate: `${DKG}kaUal`, object: UAL, graph: WS_META }, + { subject, predicate: `${DKG}assertionVersion`, object: `"${version}"^^<${XSD_INTEGER}>`, graph: WS_META }, + { subject, predicate: `${DKG}shareOperationId`, object: `"${shareId}"`, graph: WS_META }, + { subject, predicate: `${DKG}publicQuadsDigest`, object: `"${quadsDigest}"`, graph: WS_META }, + { subject, predicate: `${DKG}publicQuadsCount`, object: `"${publicCount}"^^<${XSD_INTEGER}>`, graph: WS_META }, + { subject, predicate: `${DKG}publicSnapshotRef`, object: `"${quadsDigest}"`, graph: WS_META }, + ]; + }; + + // Sixteen decoys that sort BEFORE the real operation and are rejected only + // by their triple count — exactly the rows that used to consume the window. + for (let index = 0; index < 16; index += 1) { + await store.insert(operationRows( + `aaa-decoy-${String(index).padStart(2, '0')}`, + payload.length + 1, + `sha256:decoy-${index}`, + )); + } + await store.insert(operationRows('zzz-real-operation', payload.length, digest)); + + const handler = new FinalizationHandler(store, undefined as never, { + publicSnapshotStore: snapshotStore, + }); + const verified = await (handler as unknown as { + verifyImmutableGraphScopedSnapshot(input: unknown): Promise<{ status: string } | undefined>; + }).verifyImmutableGraphScopedSnapshot({ + contextGraphId: CG, + scope, + publicTripleCount: payload.length, + expectedPublicQuadsDigest: undefined, + expectedMerkleRoot: computeFlatKCRootV10(payload, []), + ctx, + }); + + expect(verified?.status).toBe('verified'); + }); });