diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index 16e7467c64..60d5e24aa7 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -159,6 +159,8 @@ import { type SignedAgentDelegation, } from './auth/agent-delegation.js'; import { SyncVerifyWorker } from './sync-verify-worker.js'; +import type { FinalizedSwmCleanupWorker } from './finalized-swm-cleanup-worker.js'; +import type { FinalizedSwmCleanupService } from './finalized-swm-cleanup-service.js'; import { bindRandomSampling, type RandomSamplingDisabledReason, type RandomSamplingHandle, type RandomSamplingStatus } from './random-sampling-bind.js'; import { connectToMultiaddr, ensurePeerConnected as ensurePeerConnectedAtom, primeCatchupConnections as primeCatchupConnectionsAtom } from './p2p/peer-connect.js'; import { Messenger, type SloProtocolStats } from './p2p/messenger.js'; @@ -879,6 +881,24 @@ export class DKGAgentBase { */ static readonly SWM_ACK_QUORUM_TICK_MS = 5_000; + /** + * Finalized-SWM GC slice budgets. The sweep yields once either is spent and + * resumes at the next context graph in its rotation, so these trade cleanup + * latency against how long one background slice may hold store capacity. + * Env-overridable for ops tuning; the service clamps candidates to 16. + */ + static readonly FINALIZED_SWM_CLEANUP_MAX_CANDIDATES = + Math.max(1, Number(process.env['DKG_FINALIZED_SWM_CLEANUP_MAX_CANDIDATES']) || 4); + static readonly FINALIZED_SWM_CLEANUP_BUDGET_MS = + Math.max(1, Number(process.env['DKG_FINALIZED_SWM_CLEANUP_BUDGET_MS']) || 10_000); + /** + * Delay before re-waking the GC after a sweep yielded on pressure or budget, + * or drained part of a known backlog. This is the cadence that actually fires + * while the node is draining, so it is the knob ops reach for first. + */ + static readonly FINALIZED_SWM_CLEANUP_RETRY_MS = + Math.max(1, Number(process.env['DKG_FINALIZED_SWM_CLEANUP_RETRY_MS']) || 5_000); + /** * Phase B — chain-driven VM reconciliation sweep cadence. The periodic sweep * is the safety net behind the live `KnowledgeAssetRegisteredToContextGraph` @@ -964,6 +984,9 @@ export class DKGAgentBase { protected messageHandler: MessageHandler | null = null; protected chainPoller: ChainEventPoller | null = null; protected swmCleanupTimer: ReturnType | null = null; + protected finalizedSwmCleanupTimer: ReturnType | null = null; + protected finalizedSwmCleanupWorker?: FinalizedSwmCleanupWorker; + protected finalizedSwmCleanupService?: FinalizedSwmCleanupService; /** Phase B — periodic chain-driven VM reconciliation sweep timer. */ protected vmReconcileTimer: ReturnType | null = null; /** Phase B — unified per-CG coalescing and node-wide admission policy. */ diff --git a/packages/agent/src/dkg-agent-constants.ts b/packages/agent/src/dkg-agent-constants.ts index e58181cfea..6d23bbdcbe 100644 --- a/packages/agent/src/dkg-agent-constants.ts +++ b/packages/agent/src/dkg-agent-constants.ts @@ -134,6 +134,14 @@ export const SYNC_MIN_GRAPH_BUDGET_MS = 10_000; export const DEBUG_SYNC_PROGRESS = process.env.DKG_DEBUG_SYNC_PROGRESS === '1'; export const DEFAULT_SWM_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days export const SWM_CLEANUP_INTERVAL_MS = 15 * 60 * 1000; // run cleanup every 15 minutes +export const FINALIZED_SWM_CLEANUP_ROOT_PREDICATE = + 'http://dkg.io/ontology/finalizedSwmCleanupRoot'; +export const FINALIZED_SWM_CLEANUP_TASK_TYPE = + 'http://dkg.io/ontology/FinalizedSwmCleanupTask'; +export const FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE = + 'http://dkg.io/ontology/finalizedSwmCleanupMarkedAt'; +export const FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE = + 'http://dkg.io/ontology/finalizedSwmCleanupHeadFingerprint'; export const SYNC_DENIED_RESPONSE = '__DKG_SYNC_DENIED__'; // ── Gossip reconnect ────────────────────────────────────────────────── diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index c6ae75b068..cce2715120 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -98,7 +98,7 @@ import { pickNetworkTunables, withRetry, } from '@origintrail-official/dkg-core'; -import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, createTripleStore, asChangelogReader, tryReplaceGraphAtomically, type ChangelogReader, type QueryOptions, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; import { readChangelogDeltaPage } from './sync/responder/graph-plan.js'; import { decodeChangelogRequest, encodeChangelogResponse } from './sync/changelog/wire.js'; import { runChangelogSync, planPageApply } from './sync/requester/changelog-sync.js'; @@ -138,6 +138,8 @@ import { TripleStoreAsyncPromoteQueue, FileWorkspacePublicSnapshotStore, parseWorkspacePublicSnapshotNQuads, + swmKaWriteLockKey, + withKeyedLocks, type AsyncPromoteQueue, type AsyncPromoteQueueConfig, type PromoteJob, type PromoteListFilter, wrapAsRpcPreconditionIfApplicable, @@ -264,7 +266,10 @@ import { } from './sync/requester/durable-sync.js'; import { resolveSyncAgentsMeta, shouldWithholdAgentsDurableMeta } from './sync/agents-meta-policy.js'; import { runSharedMemorySync, sharedMemoryOwnershipKeyFromGraph } from './sync/requester/shared-memory-sync.js'; -import { createSharedMemorySnapshotMaterializer } from './sync/requester/swm-snapshot-materializer.js'; +import { + createSharedMemorySnapshotMaterializer, + replaceGraphScopedSwmHeadMetadata, +} from './sync/requester/swm-snapshot-materializer.js'; import { runOrderedContextGraphSyncs, type ContextGraphSyncWork, @@ -333,6 +338,12 @@ import { } from './agent-keystore.js'; import { GossipPublishHandler } from './gossip-publish-handler.js'; import { FinalizationHandler, KEEP_ROOT_COPY_PREDICATE } from './finalization-handler.js'; +import { + FinalizedSwmCleanupWorker, + type FinalizedSwmCleanupStats, + type FinalizedSwmCleanupSweepResult, +} from './finalized-swm-cleanup-worker.js'; +import { FinalizedSwmCleanupService } from './finalized-swm-cleanup-service.js'; import { reconcileContextGraph, RecentUalSet, type ChainReconcilerDeps, type OrdinalOutcome } from './chain-reconciler.js'; import { createCursorState, type CursorState } from './reconcile-cursor.js'; import { resolveStorageAckLifecycleAssetUalFromLocalSwm } from './storage-ack-lifecycle-identity.js'; @@ -368,6 +379,7 @@ type JoinApprovalRetryEntry = { nextAttemptAt: number; lastError: string; }; + import { multiaddr } from '@multiformats/multiaddr'; import { buildCclPolicyQuads, buildPolicyApprovalQuads, buildPolicyRevocationQuads, hashCclPolicy, type CclPolicyRecord, type PolicyApprovalBinding } from './ccl-policy.js'; import { CclEvaluator, parseCclPolicy, validateCclPolicy, type CclEvaluationResult, type CclFactTuple } from './ccl-evaluator.js'; @@ -886,6 +898,7 @@ type RecoverContextGraphSwmOptions = Parameters[0 interface RecoverContextGraphSwmFromPeerDependencies { store: TripleStore; + writeLocks: Map>; listSubGraphs: (contextGraphId: string) => ReturnType; createContextGraphSyncDeadline: (remainingContextGraphs: number) => number; fetchSyncPages: RecoverContextGraphSwmOptions['fetchSyncPages']; @@ -3115,15 +3128,20 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); } - // Start periodic shared memory cleanup - const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; - if (ttl > 0) { + // TTL expiry and finalized-SWM GC are deliberately independent. The GC + // worker is pressure-gated and never joined by foreground sync/finalize. + if ((this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS) > 0) { this.cleanupExpiredSharedMemory().catch(() => {}); this.swmCleanupTimer = setInterval(() => { this.cleanupExpiredSharedMemory().catch(() => {}); }, SWM_CLEANUP_INTERVAL_MS); - if (this.swmCleanupTimer.unref) this.swmCleanupTimer.unref(); + this.swmCleanupTimer.unref?.(); } + this.wakeFinalizedSwmCleanup(); + this.finalizedSwmCleanupTimer = setInterval(() => { + this.wakeFinalizedSwmCleanup(); + }, SWM_CLEANUP_INTERVAL_MS); + this.finalizedSwmCleanupTimer.unref?.(); // OT-RFC-38 LU-6: periodic reconciler that ensures the local // node is subscribed in host-mode to every locally-known @@ -5222,6 +5240,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { const recoverPrivateContextGraph = (contextGraphId: string) => runRecoverContextGraphSwmFromPeer( { store: this.store, + writeLocks: this.writeLocks, listSubGraphs: (id) => this.listSubGraphs(id), createContextGraphSyncDeadline: (remaining) => createContextGraphSyncDeadline({ remainingContextGraphs: remaining, @@ -5294,6 +5313,23 @@ export class LifecycleSyncMethods extends DKGAgentBase { return admission; }; + const insertSharedMemorySyncQuads = async (quads: readonly Quad[]) => { + // Oversize guard (OT-RFC-56): drop+tombstone protocol-violating + // literals BEFORE every peer-controlled SWM metadata insert, + // including graph-scoped head replacement, so the page cursor can + // advance instead of re-fetching the same poison row forever. + const inserted = await insertWithOversizeGuard( + (kept) => this.store.insert(kept, { + priority: 'background', + source: 'agent.sharedMemorySync.storeInsert', + }), + quads, + { recordDrops: (drops, seam) => this.oversizeTombstoneLog.record(drops, seam) }, + 'swm-sync', + ); + this.contextGraphMetaProjection.markDirtyFromQuads(inserted); + }; + const syncPublicContextGraph = (contextGraphId: string, remainingContextGraphs: number) => runSharedMemorySync({ ctx, remotePeerId, @@ -5335,22 +5371,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { store: this.store, writeLocks: this.writeLocks, invalidateListContextGraphsCache: () => this.invalidateListContextGraphsCache(), + insertReplacementMetadata: insertSharedMemorySyncQuads, }), - storeInsert: async (quads) => { - // Oversize guard (OT-RFC-56): drop+tombstone protocol-violating - // literals BEFORE insert so the SWM page cursor advances instead - // of the store throwing and the page re-fetching forever. - const inserted = await insertWithOversizeGuard( - (kept) => this.store.insert(kept, { - priority: 'background', - source: 'agent.sharedMemorySync.storeInsert', - }), - quads, - { recordDrops: (drops, seam) => this.oversizeTombstoneLog.record(drops, seam) }, - 'swm-sync', - ); - this.contextGraphMetaProjection.markDirtyFromQuads(inserted); - }, + storeInsert: insertSharedMemorySyncQuads, publicSnapshotStore: this.publicSnapshotStore, deleteCheckpoint: (key) => this.syncCheckpoints.delete(key), setCheckpoint: (key, offset) => this.syncCheckpoints.set(key, offset), @@ -5426,7 +5449,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); } - return runOrderedContextGraphSyncs({ + const summary = await runOrderedContextGraphSyncs({ work, priorities: this.config.syncContextGraphPriorities, emptyResult: emptySharedMemorySyncResult, @@ -5451,6 +5474,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { `Deferring ${item.lane} at CG ${item.contextGraphId} due to local backpressure: ${error.message}`, ), }); + return summary; }; return runSyncSingleFlight(this, singleFlightKey, runSync); @@ -5481,6 +5505,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { () => runRecoverContextGraphSwmFromPeer( { store: this.store, + writeLocks: this.writeLocks, listSubGraphs: (id) => this.listSubGraphs(id), createContextGraphSyncDeadline: (remaining) => createContextGraphSyncDeadline({ remainingContextGraphs: remaining, @@ -6041,6 +6066,11 @@ export class LifecycleSyncMethods extends DKGAgentBase { verifiedPrivateOnlyResponses: cleanDurablePrivateOnlyCompletions, }); } + if (includeSharedMemory) { + // Catch-up may nudge eventual cleanup but never waits for discovery, + // payload verification or deletion to complete. + this.wakeFinalizedSwmCleanup(); + } return { connectedPeers: stats?.totalPeers ?? peers.length, @@ -7566,31 +7596,82 @@ export class LifecycleSyncMethods extends DKGAgentBase { * and the next cleanup cycle without requiring a restart. */ setSharedMemoryTtlMs(this: DKGAgent, ttlMs: number): void { - const oldTtl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; (this.config as any).sharedMemoryTtlMs = ttlMs; - if (oldTtl <= 0 && ttlMs > 0 && !this.swmCleanupTimer) { + if (ttlMs <= 0 && this.swmCleanupTimer) { + clearInterval(this.swmCleanupTimer); + this.swmCleanupTimer = null; + } else if (ttlMs > 0 && !this.swmCleanupTimer) { this.cleanupExpiredSharedMemory().catch(() => {}); this.swmCleanupTimer = setInterval(() => { this.cleanupExpiredSharedMemory().catch(() => {}); }, SWM_CLEANUP_INTERVAL_MS); if (this.swmCleanupTimer.unref) this.swmCleanupTimer.unref(); - } else if (ttlMs <= 0 && this.swmCleanupTimer) { - clearInterval(this.swmCleanupTimer); - this.swmCleanupTimer = null; } } + getOrCreateFinalizedSwmCleanupWorker(this: DKGAgent): FinalizedSwmCleanupWorker { + if (!this.finalizedSwmCleanupWorker) { + this.finalizedSwmCleanupWorker = new FinalizedSwmCleanupWorker({ + sweep: () => this.runFinalizedSwmCleanupSweep(), + retryDelayMs: DKGAgentBase.FINALIZED_SWM_CLEANUP_RETRY_MS, + onError: (error) => { + this.log.warn( + createOperationContext('system'), + `Finalized SWM cleanup worker failed: ${error instanceof Error ? error.message : String(error)}`, + ); + }, + }); + } + return this.finalizedSwmCleanupWorker; + } + + /** Nudge independent finalized-SWM GC without joining its store work. */ + wakeFinalizedSwmCleanup(this: DKGAgent): void { + this.getOrCreateFinalizedSwmCleanupWorker().wake(); + } + + getFinalizedSwmCleanupStats(this: DKGAgent): FinalizedSwmCleanupStats { + return this.getOrCreateFinalizedSwmCleanupWorker().snapshot(); + } + + getOrCreateFinalizedSwmCleanupService(this: DKGAgent): FinalizedSwmCleanupService { + if (!this.finalizedSwmCleanupService) { + this.finalizedSwmCleanupService = new FinalizedSwmCleanupService({ + store: this.store, + writeLocks: this.writeLocks, + eventBus: this.eventBus, + // Deliberately ignores the sweep's query options: this is the shared, + // TTL-cached listing, so its result is normally amortized across + // foreground callers rather than store work attributable to the GC, and + // threading the sweep's deadline signal in would let a foreground caller + // joining the same in-flight promise inherit the GC's abort. It stays + // the id source because it is the only one that resolves owner/name + // context graphs — a graph-URI walk cannot tell `/` apart + // from a sub-graph. The per-context-graph enumeration below is the call + // that scales with graph count, and that one is lane-tagged. + listContextGraphIds: async () => (await this.listContextGraphs()).map((row) => row.id), + listSharedMemoryMetaGraphs: (contextGraphId, options) => + listSharedMemoryMetaGraphs(this.store, contextGraphId, options), + maxCandidatesPerSweep: DKGAgentBase.FINALIZED_SWM_CLEANUP_MAX_CANDIDATES, + wallClockBudgetMs: DKGAgentBase.FINALIZED_SWM_CLEANUP_BUDGET_MS, + }); + } + return this.finalizedSwmCleanupService; + } + + /** Run one small idle-only GC slice in the dedicated cleanup service. */ + async runFinalizedSwmCleanupSweep(this: DKGAgent): Promise { + return this.getOrCreateFinalizedSwmCleanupService().runSweep(); + } + /** - * Remove expired shared memory operations and their data. - * Queries SWM meta for operations with publishedAt older than the TTL, - * deletes the corresponding triples from shared memory and SWM meta, - * and removes the root entities from workspaceOwnedEntities. + * Remove expired shared-memory operations. Finalized-SWM lifecycle GC is + * owned exclusively by FinalizedSwmCleanupService and its scheduler above. */ async cleanupExpiredSharedMemory(this: DKGAgent): Promise { const ttl = this.config.sharedMemoryTtlMs ?? DEFAULT_SWM_TTL_MS; if (ttl <= 0) return 0; - const ctx = createOperationContext('share'); const cutoff = new Date(Date.now() - ttl).toISOString(); let totalDeleted = 0; @@ -7613,7 +7694,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { // Each meta graph describes exactly one SWM data bucket: // `…/_shared_memory_meta` ↔ `…/_shared_memory` (root or per-subgraph). const wsGraph = wsMetaGraph.slice(0, -'_meta'.length); - const expiredOps = await this.store.query( `SELECT ?op WHERE { GRAPH <${wsMetaGraph}> { @@ -7775,10 +7855,14 @@ async function listGraphFamily(store: TripleStore, rootGraph: string): Promise { +async function listGraphsByPrefix( + store: TripleStore, + prefix: string, + options?: QueryOptions, +): Promise { return store.listGraphsByPrefix - ? store.listGraphsByPrefix(prefix) - : (await store.listGraphs()).filter((graph) => graph.startsWith(prefix)); + ? store.listGraphsByPrefix(prefix, options) + : (await store.listGraphs(options)).filter((graph) => graph.startsWith(prefix)); } async function runRecoverContextGraphSwmFromPeer( @@ -7790,6 +7874,23 @@ async function runRecoverContextGraphSwmFromPeer( const admission = await getSharedMemorySubGraphAdmission( dependencies.store, contextGraphId, dependencies.listSubGraphs(contextGraphId), ); + const insertRecoveredSwmQuads = async (quads: readonly Quad[]) => { + // Oversize guard (OT-RFC-56) — recovered rows and graph-scoped + // replacement metadata are both peer-controlled data. + const inserted = await insertWithOversizeGuard( + (kept) => dependencies.store.insert(kept, { + priority: 'background', + source: 'agent.swmRecovery.insert', + }), + quads, + { recordDrops: (drops, seam) => dependencies.recordDrops(drops, seam) }, + 'swm-recovery', + ); + if (inserted.length > 0) { + dependencies.invalidateListContextGraphsCache(); + dependencies.markMetaProjectionDirty(inserted); + } + }; return recoverContextGraphSwm({ ctx, remotePeerId, @@ -7808,22 +7909,7 @@ async function runRecoverContextGraphSwmFromPeer( // dirty on insert (parity with runSharedMemorySync's // insertSyncedQuadsAndInvalidateListCache); deletes pass through to the store. store: { - insert: async (quads) => { - // Oversize guard (OT-RFC-56) — recovered rows are peer data too. - const inserted = await insertWithOversizeGuard( - (kept) => dependencies.store.insert(kept, { - priority: 'background', - source: 'agent.swmRecovery.insert', - }), - quads, - { recordDrops: (drops, seam) => dependencies.recordDrops(drops, seam) }, - 'swm-recovery', - ); - if (inserted.length > 0) { - dependencies.invalidateListContextGraphsCache(); - dependencies.markMetaProjectionDirty(inserted); - } - }, + insert: insertRecoveredSwmQuads, replaceGraph: async (graph, quads) => { const replaced = await tryReplaceGraphAtomically( dependencies.store, @@ -7909,39 +7995,17 @@ async function runRecoverContextGraphSwmFromPeer( }, replaceMetaForGraphAssets: async (assets) => { for (const asset of assets) { - const linkedOperations = await dependencies.store.query( - `SELECT DISTINCT ?op WHERE { GRAPH <${assertSafeIri(asset.metaGraph)}> { ` + - `<${assertSafeIri(asset.headSubject)}> ?shareId . ` + - `?op ?shareId ; ` + - ` <${assertSafeIri(asset.kaUal)}> . } }`, - { - priority: 'background', - source: 'agent.swmRecovery.replaceMetaForGraphAssets.findOperations', - }, - ); - const operationSubjects = new Set([asset.operationSubject]); - if (linkedOperations.type === 'bindings') { - for (const row of linkedOperations.bindings) { - const operation = row['op']; - if (operation) operationSubjects.add(operation); - } - } - await dependencies.store.deleteByPattern( - { graph: asset.metaGraph, subject: asset.headSubject }, - { - priority: 'background', - source: 'agent.swmRecovery.replaceMetaForGraphAssets.deleteHead', - }, + await withKeyedLocks( + dependencies.writeLocks, + [swmKaWriteLockKey(contextGraphId, asset.subGraphName, asset.kaUal)], + () => replaceGraphScopedSwmHeadMetadata({ + store: dependencies.store, + contextGraphId, + descriptor: asset, + sourcePrefix: 'agent.swmRecovery.replaceMetaForGraphAssets', + insertReplacementMetadata: insertRecoveredSwmQuads, + }), ); - for (const operationSubject of operationSubjects) { - await dependencies.store.deleteByPattern( - { graph: asset.metaGraph, subject: operationSubject }, - { - priority: 'background', - source: 'agent.swmRecovery.replaceMetaForGraphAssets.deleteOperation', - }, - ); - } } }, ensureContextGraph: async (cgId) => { @@ -8005,12 +8069,16 @@ async function isKnownContextGraphUri(store: TripleStore, contextGraphUri: strin * families such as `…/_verifiable_memory/…` or `…/_shared_memory_snapshots/…` * can never be misread as a sub-graph meta graph. */ -async function listSharedMemoryMetaGraphs(store: TripleStore, contextGraphId: string): Promise { +async function listSharedMemoryMetaGraphs( + store: TripleStore, + contextGraphId: string, + options?: QueryOptions, +): Promise { const rootMetaGraph = contextGraphWorkspaceMetaGraphUri(contextGraphId); const cgPrefix = `did:dkg:context-graph:${contextGraphId}/`; const metaSuffix = '/_shared_memory_meta'; const metaGraphs = [rootMetaGraph]; - for (const graph of await listGraphsByPrefix(store, cgPrefix)) { + for (const graph of await listGraphsByPrefix(store, cgPrefix, options)) { if (graph === rootMetaGraph || !graph.endsWith(metaSuffix)) continue; const subGraphName = graph.slice(cgPrefix.length, graph.length - metaSuffix.length); if (!validateSubGraphName(subGraphName).valid) continue; diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 4893d23842..dc84d9e25e 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -1654,6 +1654,11 @@ export class SwmSubstrateMethods extends DKGAgentBase { markContextGraphMetaDirtyFromQuads: (quads) => { this.contextGraphMetaProjection.markDirtyFromQuads(quads); }, + // The SAME map catch-up and the finalized-SWM GC take, so the cleanup + // marker write serializes against them on the per-KA key. + writeLocks: this.writeLocks, + publicSnapshotStore: this.publicSnapshotStore, + wakeFinalizedSwmCleanup: () => this.wakeFinalizedSwmCleanup(), runtime: this.finalizationRuntime, }, ); diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 85f8ff7c3f..314436f56a 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -1642,6 +1642,15 @@ export class DKGAgent extends DKGAgentBase { clearInterval(this.swmCleanupTimer); this.swmCleanupTimer = null; } + if (this.finalizedSwmCleanupTimer) { + clearInterval(this.finalizedSwmCleanupTimer); + this.finalizedSwmCleanupTimer = null; + } + if (this.finalizedSwmCleanupWorker) { + await this.finalizedSwmCleanupWorker.close(); + this.finalizedSwmCleanupWorker = undefined; + } + this.finalizedSwmCleanupService = undefined; if (this.hostModeReconcilerTimer) { clearInterval(this.hostModeReconcilerTimer); this.hostModeReconcilerTimer = null; diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index a63039c1f6..0001986f62 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -26,6 +26,7 @@ import { tryReplaceGraphAndSubjectAtomically, StoreSchedulerBusyError, type GraphWriteGenSource, + type QueryOptions, type SharedMemoryResultBudget, type SwmKaGraphBound, type TripleStore, @@ -46,10 +47,15 @@ import { shouldApplyMaterialization, writeMaterializedVersion, materializedVersionQuad, withMaterializationLock, KnowledgeAssetWorkspaceHeadCorruptError, + resolveKnowledgeAssetOperationPublicQuads, resolveKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, + workspaceOperationSubject, workspacePublicQuadsDigest, type MaterializedVersion, type KnowledgeAssetWorkspaceHead, + type WorkspacePublicSnapshotStore, type KCMetadata, type KAMetadata, type OnChainProvenance, } from '@origintrail-official/dkg-publisher'; const DKG_NS = 'http://dkg.io/ontology/'; @@ -93,6 +99,23 @@ import { type VerifiedGraphScopedFinalizationEvidence, } from './finalization-graph-envelope.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; +import { + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, +} from './dkg-agent-constants.js'; +export { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; +import { + buildFinalizedSwmCleanupTaskQuads, +} from './finalized-swm-cleanup-marker.js'; +import { + verifyExactGraphScopedLayer, + type ExactGraphScopedLayerVerification, +} from './graph-scoped-layer-verification.js'; /** * Predicate for the durable per-root keep-root-copy signal the publisher @@ -177,27 +200,6 @@ function equalBytes(left: Uint8Array, right: Uint8Array): boolean { && left.every((byte, index) => byte === right[index]); } -type ExactGraphScopedLayerVerification = - | { - status: 'verified'; - graphUri: string; - quads: Quad[]; - merkleRoot: Uint8Array; - } - | { - status: 'count-mismatch'; - graphUri: string; - actualCount: number; - } - | { - status: 'merkle-mismatch'; - graphUri: string; - } - | { - status: 'head-mismatch'; - graphUri: string; - }; - type GraphScopedMaterializationEnvelope = Pick< KnowledgeAssetWorkspaceHead, | 'publicTripleCount' @@ -292,6 +294,15 @@ export interface FinalizationHandlerOptions { eventBus?: EventBus; resolveContextGraphOnChainId?: ResolveContextGraphOnChainId; markContextGraphMetaDirtyFromQuads?: MarkContextGraphMetaDirtyFromQuads; + /** + * The SAME per-KA SWM writer lock map catch-up and the finalized-SWM GC use. + * Taken around the cleanup-marker write so it cannot land inside catch-up's + * read-delete-reinsert of the operation subject. Absent => that write is + * unserialized; see `markFinalizedGraphScopedSwmForCleanup`. + */ + writeLocks?: Map>; + publicSnapshotStore?: WorkspacePublicSnapshotStore; + wakeFinalizedSwmCleanup?: () => void; lifecycleLogOptions?: FinalizationLifecycleLogOptions; recoveryStore?: FinalizationRecoveryStore; runtime?: FinalizationRuntime; @@ -361,6 +372,9 @@ export class FinalizationHandler { private readonly eventBus: EventBus | undefined; private readonly resolveContextGraphOnChainId: ResolveContextGraphOnChainId | undefined; private readonly markContextGraphMetaDirtyFromQuads: MarkContextGraphMetaDirtyFromQuads | undefined; + private readonly writeLocks: Map> | undefined; + private readonly publicSnapshotStore: WorkspacePublicSnapshotStore | undefined; + private readonly wakeFinalizedSwmCleanup: (() => void) | undefined; private readonly recovery: FinalizationRecovery; private readonly log = new Logger('FinalizationHandler'); private readonly lifecycle: FinalizationLifecycleLogger; @@ -420,6 +434,9 @@ export class FinalizationHandler { this.eventBus = options.eventBus; this.resolveContextGraphOnChainId = options.resolveContextGraphOnChainId; this.markContextGraphMetaDirtyFromQuads = options.markContextGraphMetaDirtyFromQuads; + this.writeLocks = options.writeLocks; + this.publicSnapshotStore = options.publicSnapshotStore; + this.wakeFinalizedSwmCleanup = options.wakeFinalizedSwmCleanup; this.lifecycle = new FinalizationLifecycleLogger( this.log, options.runtime ?? options.lifecycleLogOptions, @@ -1083,15 +1100,17 @@ export class FinalizationHandler { }); let layerVerification = vmVerification; if (layerVerification.status !== 'verified') { - layerVerification = await this.verifyExactGraphScopedLayer({ + layerVerification = await this.resolveVerifiedGraphScopedFinalizationSource({ contextGraphId, scope, - layer: MemoryLayer.SharedWorkingMemory, publicTripleCount, privateMerkleRoot, expectedMerkleRoot: msg.kcMerkleRoot, expectedPublicQuadsDigest: head.publicQuadsDigest, + expectedHead: head, subGraphName, + allowImmutableSnapshot: true, + ctx, }); if (layerVerification.status === 'count-mismatch') { this.log.warn( @@ -1187,6 +1206,14 @@ export class FinalizationHandler { subGraphName, }); if (metadataState === 'matching') { + await this.markFinalizedGraphScopedSwmForCleanup({ + contextGraphId, + scope, + expectedHead: head, + expectedMerkleRoot: msg.kcMerkleRoot, + subGraphName, + ctx, + }); this.markProcessed(dedupeKey); this.log.info(ctx, `Finalization: graph-scoped KA ${scope.ual} is already confirmed`); return 'already-confirmed'; @@ -1218,6 +1245,16 @@ export class FinalizationHandler { this.log.info(ctx, `Finalization: newer graph-scoped assertion already materialized for ${scope.ual}`); return 'already-confirmed'; } + if (outcome === 'applied') { + await this.markFinalizedGraphScopedSwmForCleanup({ + contextGraphId, + scope, + expectedHead: head, + expectedMerkleRoot: msg.kcMerkleRoot, + subGraphName, + ctx, + }); + } this.markProcessed(dedupeKey); this.log.info( @@ -1317,37 +1354,330 @@ export class FinalizationHandler { expectedMerkleRoot: Uint8Array; expectedPublicQuadsDigest?: string; subGraphName?: string; + queryOptions?: QueryOptions; + }): Promise { + return verifyExactGraphScopedLayer({ + store: this.store, + ...input, + queryOptions: { + source: 'agent.finalization.verifyExactLayer', + ...input.queryOptions, + }, + }); + } + + /** + * Rebuild a finalized source from its immutable operation snapshot when the + * active SWM graph has already been drained. This keeps durable receipt/reorg + * recovery independent from retaining a second live copy of the payload. + * + * Once the idle GC has done its job this is the NORMAL path for a late + * receipt, not an exception, so its cost has to be held down — but this is a + * SEARCH, and the only sound way to make a search cheaper is to remove work + * that provably cannot change its answer: + * + * - Discovery stays a bounded metadata read (one bound `kaUal`, LIMIT 16) + * and keeps the receipt lane's default priority. Receipts are + * latency-sensitive; demoting them to the background lane would trade a + * CPU spike for receipt starvation under load, which is the worse failure. + * - Payload verification is the expensive half — a full snapshot read plus + * digest plus Merkle root per candidate — and it is deduplicated BY + * CONTENT (see the memo below), not truncated by candidate count. When the + * evidence carries a digest, every candidate advertises that same digest, + * so the memo collapses the whole list to one payload read. When it does + * not (`VerifiedGraphScopedFinalizationEvidence.publicQuadsDigest` is + * optional), the candidates genuinely differ and each one has to be + * checked; that residue is inherent to the search and stays bounded by the + * discovery LIMIT. + */ + private async verifyImmutableGraphScopedSnapshot(input: { + contextGraphId: string; + scope: ReturnType; + expectedHead?: KnowledgeAssetWorkspaceHead; + publicTripleCount: number; + expectedPublicQuadsDigest?: string; + privateMerkleRoot?: Uint8Array; + expectedMerkleRoot: Uint8Array; + subGraphName?: string; + ctx: OperationContext; + }): Promise | undefined> { + const graphManager = new GraphManager(this.store); + const shareOperationIds: Array<{ shareOperationId: string; digest?: string }> = []; + if (input.expectedHead) { + shareOperationIds.push({ shareOperationId: input.expectedHead.shareOperationId }); + } else { + const metaGraph = graphManager.sharedMemoryMetaUri( + input.contextGraphId, + input.subGraphName, + ); + // Every DISCRIMINATING test must run inside the query, ahead of LIMIT. + // With only the version filter here, the limit truncated the CANDIDATE + // SET rather than the work: sixteen same-version operations with the + // wrong triple count filled the window, the matching one was never + // returned, and since `ORDER BY ?shareId` is deterministic over stable + // store state, every retry re-derived the identical sixteen and missed + // the same snapshot permanently. A KA re-shared repeatedly at one version + // reaches that without anything unusual happening. + // + // `?count` is compared numerically first, so a non-canonical typed + // literal ("02"^^xsd:integer) still matches, with the lexical form as a + // fallback for an untyped one. The JS check below parses either, and this + // filter must never be STRICTER than it — a filter that rejects a + // candidate the caller would have accepted is the same missed-discovery + // bug wearing different clothes. + const countFilter = Number.isSafeInteger(input.publicTripleCount) + ? `\n FILTER(?count = ${input.publicTripleCount}` + + ` || STR(?count) = ${JSON.stringify(String(input.publicTripleCount))})` + : ''; + const digestFilter = input.expectedPublicQuadsDigest === undefined + ? '' + : `\n FILTER(STR(?digest) = ${JSON.stringify(input.expectedPublicQuadsDigest)})`; + const result = await this.store.query( + `SELECT DISTINCT ?shareId ?digest ?count WHERE { + GRAPH <${assertSafeIri(metaGraph)}> { + ?operation <${DKG_NS}contentScopeVersion> 2 ; + <${DKG_NS}kaUal> <${assertSafeIri(input.scope.ual)}> ; + <${DKG_NS}assertionVersion> ?version ; + <${DKG_NS}shareOperationId> ?shareId ; + <${DKG_NS}publicQuadsDigest> ?digest ; + <${DKG_NS}publicQuadsCount> ?count . + FILTER(STR(?version) = ${JSON.stringify(input.scope.assertionVersion)})${countFilter}${digestFilter} + } + } ORDER BY ?shareId LIMIT 16`, + { source: 'agent.finalization.verifyImmutableSnapshot' }, + ); + if (result.type === 'bindings') { + for (const row of result.bindings) { + const shareId = stripOptionalLiteral(row['shareId']); + const digest = stripOptionalLiteral(row['digest']); + const count = Number.parseInt(stripOptionalLiteral(row['count']) ?? '', 10); + if ( + shareId + && count === input.publicTripleCount + && ( + input.expectedPublicQuadsDigest === undefined + || digest === input.expectedPublicQuadsDigest + ) + ) { + shareOperationIds.push({ shareOperationId: shareId, ...(digest ? { digest } : {}) }); + } + } + } + } + + const seenShareOperationIds = new Set(); + const candidates = shareOperationIds.filter((candidate) => { + if (seenShareOperationIds.has(candidate.shareOperationId)) return false; + seenShareOperationIds.add(candidate.shareOperationId); + return true; + }); + // Content-level memo. `resolveKnowledgeAssetOperationPublicQuads` throws + // unless the resolved payload hashes to the digest stored on that operation + // subject, so two candidates advertising the same digest resolve to the + // same quads and therefore to the same count and Merkle root. Once one of + // them has been fully resolved and rejected, every other candidate with + // that digest is provably a repeat of work already done — skipping it + // cannot change the answer. A THROW must not memo: that means we could not + // read THAT operation's snapshot, not that the content is wrong, and a + // sibling operation may still hold a readable copy. + // + // This bounds the repeated work without bounding the SEARCH. A cap on the + // candidate count would do the opposite: with `ORDER BY ?shareId` the order + // is unrelated to which candidate matches, so truncating it does not do + // less work, it returns a different answer — `undefined`, which decays into + // 'no-swm'. That outcome leaves the cursor for a sweep retry, but the + // candidate list is deterministic, so every retry re-derives the identical + // truncation and the receipt is stranded rather than delayed. + const rejectedDigests = new Set(); + for (const { shareOperationId, digest: candidateDigest } of candidates) { + if (candidateDigest !== undefined && rejectedDigests.has(candidateDigest)) continue; + let quads: Quad[]; + try { + const snapshot = await resolveKnowledgeAssetOperationPublicQuads({ + store: this.store, + graphManager, + contextGraphId: input.contextGraphId, + shareOperationId, + kaUal: input.scope.ual, + assertionVersion: input.scope.assertionVersion, + subGraphName: input.subGraphName, + publicSnapshotStore: this.publicSnapshotStore, + queryOptions: { source: 'agent.finalization.resolveImmutableSnapshotPayload' }, + }); + quads = snapshot.quads.map((quad) => ({ ...quad, graph: '' })); + } catch (error) { + if (error instanceof StoreSchedulerBusyError) throw error; + continue; + } + if ( + quads.length !== input.publicTripleCount + || ( + input.expectedPublicQuadsDigest !== undefined + && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest + ) + ) { + if (candidateDigest !== undefined) rejectedDigests.add(candidateDigest); + continue; + } + const merkleRoot = computeFlatKCRoot( + quads, + input.privateMerkleRoot ? [input.privateMerkleRoot] : [], + ); + if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { + if (candidateDigest !== undefined) rejectedDigests.add(candidateDigest); + continue; + } + return { + status: 'verified', + graphUri: knowledgeAssetLayerGraphUri( + input.contextGraphId, + MemoryLayer.SharedWorkingMemory, + input.scope, + input.subGraphName, + ), + quads, + merkleRoot, + }; + } + this.log.warn( + input.ctx, + `Finalization: no immutable graph-scoped snapshot matches ${input.scope.ual}`, + ); + return undefined; + } + + /** + * Resolve one exact finalization source from the live SWM graph first and + * then, when receipt-backed recovery permits it, the immutable operation + * snapshot. Every finalization entrypoint shares this order and validation. + */ + private async resolveVerifiedGraphScopedFinalizationSource(input: { + contextGraphId: string; + scope: ReturnType; + publicTripleCount: number; + privateMerkleRoot?: Uint8Array; + expectedMerkleRoot: Uint8Array; + expectedPublicQuadsDigest?: string; + expectedHead?: KnowledgeAssetWorkspaceHead; + subGraphName?: string; + allowImmutableSnapshot: boolean; + ctx: OperationContext; }): Promise { - const graphUri = knowledgeAssetLayerGraphUri( + const liveVerification = await this.verifyExactGraphScopedLayer({ + contextGraphId: input.contextGraphId, + scope: input.scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: input.publicTripleCount, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + expectedPublicQuadsDigest: input.expectedPublicQuadsDigest, + subGraphName: input.subGraphName, + }); + if (liveVerification.status === 'verified' || !input.allowImmutableSnapshot) { + return liveVerification; + } + const snapshotVerification = await this.verifyImmutableGraphScopedSnapshot({ + contextGraphId: input.contextGraphId, + scope: input.scope, + expectedHead: input.expectedHead, + publicTripleCount: input.publicTripleCount, + expectedPublicQuadsDigest: input.expectedPublicQuadsDigest, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + subGraphName: input.subGraphName, + ctx: input.ctx, + }); + return snapshotVerification ?? liveVerification; + } + + /** + * Persist one constant-size cleanup task plus an immutable operation + * tombstone. This is the ONLY finalized-SWM work performed in the foreground: + * no payload read, graph hash, cleanup discovery, or deletion is awaited. + * + * The task subject is independent from the mutable workspace head, so a late + * snapshot replacement cannot erase the worker's durable backlog entry. The + * operation tombstone survives task retirement so a late snapshot can re-arm + * the independent task before it materializes. Ingest still performs no + * discovery, graph verification, or deletion. + */ + private async markFinalizedGraphScopedSwmForCleanup(input: { + contextGraphId: string; + scope: ReturnType; + expectedHead: KnowledgeAssetWorkspaceHead; + expectedMerkleRoot: Uint8Array; + subGraphName?: string; + ctx: OperationContext; + }): Promise<'marked' | 'preserved'> { + const graphManager = new GraphManager(this.store); + const metaGraph = graphManager.sharedMemoryMetaUri( input.contextGraphId, - input.layer, - input.scope, input.subGraphName, ); - const result = await this.store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(graphUri)}> { ?s ?p ?o } }`, - { source: 'agent.finalization.verifyExactLayer' }, - ); - const quads = result.type === 'quads' - ? result.quads.map((quad) => ({ ...quad, graph: '' })) - : []; - if (quads.length !== input.publicTripleCount) { - return { status: 'count-mismatch', graphUri, actualCount: quads.length }; - } - const merkleRoot = computeFlatKCRoot( - quads, - input.privateMerkleRoot ? [input.privateMerkleRoot] : [], + const operationSubject = workspaceOperationSubject( + input.contextGraphId, + input.expectedHead.shareOperationId, ); - if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { - return { status: 'merkle-mismatch', graphUri }; - } - if ( - input.expectedPublicQuadsDigest !== undefined - && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest - ) { - return { status: 'head-mismatch', graphUri }; - } - return { status: 'verified', graphUri, quads, merkleRoot }; + const cleanupRootHex = ethers.hexlify(input.expectedMerkleRoot).toLowerCase(); + const cleanupRoot = JSON.stringify(cleanupRootHex); + const markedAtIso = new Date().toISOString(); + const markedAt = `"${markedAtIso}"^^`; + const writeMarker = () => this.store.insert([ + ...buildFinalizedSwmCleanupTaskQuads({ + contextGraphId: input.contextGraphId, + subGraphName: input.subGraphName, + head: input.expectedHead, + expectedMerkleRootHex: cleanupRootHex, + metaGraph, + markedAtIso, + }), + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: cleanupRoot, + graph: metaGraph, + }, + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: markedAt, + graph: metaGraph, + }, + ]); + // Serialize against the other writers of this operation subject. The + // tombstone lands on `operationSubject`, which catch-up REPLACES under this + // same lock: it reads the tombstone, then deletes the subject, then + // re-inserts from that snapshot. An unlocked marker write can land inside + // that window — the marker runs at default priority while the replace's + // reads and deletes are queued at background, so it is admitted ahead of + // them — and the re-insert then restores a snapshot taken before the marker + // existed. The tombstone is gone, the independent task survives, the GC + // cleans the lifecycle once and retires the task, and the next catch-up + // re-materializes the SWM copy with nothing left to re-arm cleanup. That is + // a PERMANENT resurrection of exactly what this component exists to remove. + // + // This is compatible with keeping finalization foreground work O(1): the + // constraint is on WORK, not on waiting. This path already read and hashed + // the entire SWM payload via `verifyExactGraphScopedLayer` before reaching + // here, so a bounded wait behind one KA's replace is smaller than what it + // has already done, and it adds no cleanup discovery, verification, + // deletion, or GC wait. The insert is a leaf operation, so taking the lock + // here cannot nest and cannot deadlock. + if (this.writeLocks) { + await withKeyedLocks( + this.writeLocks, + [swmKaWriteLockKey(input.contextGraphId, input.subGraphName, input.scope.ual)], + writeMarker, + ); + } else { + // No shared lock map wired (test/embedded construction). Still mark — + // refusing would leave the finalized copy with no cleanup record at all, + // which is worse than an unserialized write — but this path is NOT + // protected against the interleaving above. + await writeMarker(); + } + this.wakeFinalizedSwmCleanup?.(); + return 'marked'; } /** Recognize exact confirmed VM state from surviving immutable metadata. */ @@ -1552,6 +1882,23 @@ export class FinalizationHandler { this.log.warn(ctx, `Chain-reconcile: invalid private commitment for graph-scoped KA ${ual}`); return 'no-swm'; } + const markMatchedWorkspaceHeadForCleanup = async (): Promise => { + if ( + !workspaceHead + || preserveNewerWorkspaceLifecycle + || workspaceHead.assertionVersion !== scope.assertionVersion + ) { + return; + } + await this.markFinalizedGraphScopedSwmForCleanup({ + contextGraphId, + scope, + expectedHead: workspaceHead, + expectedMerkleRoot: merkleRoot, + subGraphName, + ctx, + }); + }; const vmVerification = await this.verifyExactGraphScopedLayer({ contextGraphId, scope, @@ -1588,6 +1935,7 @@ export class FinalizationHandler { scope, materializedVersion, }); + await markMatchedWorkspaceHeadForCleanup(); this.log.info(ctx, `Chain-reconcile: ${ual} already has exact VM content and metadata`); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } @@ -1656,21 +2004,26 @@ export class FinalizationHandler { ); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } + await markMatchedWorkspaceHeadForCleanup(); this.log.info(ctx, `Chain-reconcile: exact VM graph already matches ${ual}; repaired metadata`); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } - const swmVerification = await this.verifyExactGraphScopedLayer({ + const swmVerification = await this.resolveVerifiedGraphScopedFinalizationSource({ contextGraphId, scope, - layer: MemoryLayer.SharedWorkingMemory, publicTripleCount: head.publicTripleCount, privateMerkleRoot, expectedMerkleRoot: merkleRoot, expectedPublicQuadsDigest: trustedAssertionEvidence ? trustedAssertionEvidence.publicQuadsDigest : workspaceHead?.publicQuadsDigest, + expectedHead: workspaceHead?.assertionVersion === scope.assertionVersion + ? workspaceHead + : undefined, subGraphName, + allowImmutableSnapshot: trustedAssertionEvidence !== undefined, + ctx, }); if (swmVerification.status === 'count-mismatch') { this.log.info( @@ -1735,6 +2088,7 @@ export class FinalizationHandler { ); return preserveNewerWorkspaceLifecycle ? 'stale-target' : 'already-confirmed'; } + await markMatchedWorkspaceHeadForCleanup(); this.log.info( ctx, `Chain-reconcile: promoted exact graph-scoped SWM assertion to VM for ${ual} (ka=${kaId})`, diff --git a/packages/agent/src/finalized-swm-cleanup-marker.ts b/packages/agent/src/finalized-swm-cleanup-marker.ts new file mode 100644 index 0000000000..759b9224fe --- /dev/null +++ b/packages/agent/src/finalized-swm-cleanup-marker.ts @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from 'node:crypto'; +import { GRAPH_KA_CONTENT_SCOPE_VERSION } from '@origintrail-official/dkg-core'; +import type { Quad } from '@origintrail-official/dkg-storage'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; + +const DKG = 'http://dkg.io/ontology/'; +const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; +const XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer'; +const XSD_DATE_TIME = 'http://www.w3.org/2001/XMLSchema#dateTime'; + +/** Stable fields that identify one exact graph-scoped SWM lifecycle. */ +export interface FinalizedSwmCleanupHeadIdentity { + kaUal: string; + assertionVersion: bigint | number | string; + assertionGraph: string; + publicQuadsDigest: string; + publicTripleCount: number; + privateMerkleRoot?: string; + privateTripleCount: number; + shareOperationId: string; + publisherPeerId: string; + accessPolicy?: string; + allowedPeers: readonly string[]; +} + +export function finalizedSwmCleanupHeadFingerprint( + head: FinalizedSwmCleanupHeadIdentity, +): string { + return createHash('sha256').update(JSON.stringify({ + kaUal: head.kaUal, + assertionVersion: head.assertionVersion.toString(), + assertionGraph: head.assertionGraph, + publicQuadsDigest: head.publicQuadsDigest, + publicTripleCount: head.publicTripleCount, + privateMerkleRoot: head.privateMerkleRoot?.toLowerCase(), + privateTripleCount: head.privateTripleCount, + shareOperationId: head.shareOperationId, + publisherPeerId: head.publisherPeerId, + accessPolicy: head.accessPolicy, + allowedPeers: [...head.allowedPeers].sort(), + })).digest('hex'); +} + +export function finalizedSwmCleanupTaskSubject(input: { + contextGraphId: string; + subGraphName?: string; + head: FinalizedSwmCleanupHeadIdentity; +}): string { + const key = JSON.stringify([ + input.contextGraphId, + input.subGraphName ?? '', + input.head.kaUal, + input.head.assertionVersion.toString(), + input.head.shareOperationId, + ]); + return `urn:dkg:finalized-swm-cleanup:${createHash('sha256').update(key).digest('hex')}`; +} + +/** Build the durable, constant-size work item owned by the idle GC worker. */ +export function buildFinalizedSwmCleanupTaskQuads(input: { + contextGraphId: string; + subGraphName?: string; + head: FinalizedSwmCleanupHeadIdentity; + expectedMerkleRootHex: string; + metaGraph: string; + markedAtIso: string; +}): Quad[] { + const taskSubject = finalizedSwmCleanupTaskSubject(input); + const integer = (value: bigint | number | string) => + `"${String(value)}"^^<${XSD_INTEGER}>`; + return [ + { subject: taskSubject, predicate: RDF_TYPE, object: FINALIZED_SWM_CLEANUP_TASK_TYPE, graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}contentScopeVersion`, object: integer(GRAPH_KA_CONTENT_SCOPE_VERSION), graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}kaUal`, object: input.head.kaUal, graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}assertionVersion`, object: integer(input.head.assertionVersion), graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}shareOperationId`, object: JSON.stringify(input.head.shareOperationId), graph: input.metaGraph }, + { subject: taskSubject, predicate: `${DKG}assertionGraph`, object: input.head.assertionGraph, graph: input.metaGraph }, + { subject: taskSubject, predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, object: JSON.stringify(input.expectedMerkleRootHex.toLowerCase()), graph: input.metaGraph }, + { subject: taskSubject, predicate: FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, object: JSON.stringify(finalizedSwmCleanupHeadFingerprint(input.head)), graph: input.metaGraph }, + { subject: taskSubject, predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, object: `"${input.markedAtIso}"^^<${XSD_DATE_TIME}>`, graph: input.metaGraph }, + ...(input.subGraphName ? [{ + subject: taskSubject, + predicate: `${DKG}subGraphName`, + object: JSON.stringify(input.subGraphName), + graph: input.metaGraph, + }] : []), + ]; +} diff --git a/packages/agent/src/finalized-swm-cleanup-service.ts b/packages/agent/src/finalized-swm-cleanup-service.ts new file mode 100644 index 0000000000..21597cfc01 --- /dev/null +++ b/packages/agent/src/finalized-swm-cleanup-service.ts @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { + DKGEvent, + Logger, + MemoryLayer, + assertSafeIri, + contextGraphDataUri, + createGraphKnowledgeAssetScope, + createOperationContext, + type EventBus, +} from '@origintrail-official/dkg-core'; +import { + GraphManager, + StoreSchedulerBusyError, + asGraphWriteGenSource, + tryReplaceGraphAndSubjectAtomically, + type GraphWriteGenSource, + type QueryOptions, + type StorePressureSnapshot, + type TripleStore, +} from '@origintrail-official/dkg-storage'; +import { + KnowledgeAssetWorkspaceHeadCorruptError, + resolveKnowledgeAssetWorkspaceHead, + sameKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, + workspaceKnowledgeAssetHeadSubject, + type KnowledgeAssetWorkspaceHead, +} from '@origintrail-official/dkg-publisher'; +import { ethers } from 'ethers'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from './dkg-agent-constants.js'; +import { finalizedSwmCleanupHeadFingerprint } from './finalized-swm-cleanup-marker.js'; +import type { FinalizedSwmCleanupSweepResult } from './finalized-swm-cleanup-worker.js'; +import { verifyExactGraphScopedLayer } from './graph-scoped-layer-verification.js'; + +function stripOptionalLiteral(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const typedLiteral = value.match(/^"([\s\S]*)"\^\^<[^>]+>$/); + if (typedLiteral) return typedLiteral[1]; + const plainLiteral = value.match(/^"([\s\S]*)"$/); + return plainLiteral?.[1] ?? value; +} + +function hasActiveStorePressure(snapshot: StorePressureSnapshot | undefined): boolean { + if (!snapshot) return false; + return [ + snapshot.ackInflight, + snapshot.healthInflight ?? 0, + snapshot.normalInflight, + snapshot.backgroundInflight, + snapshot.ackQueued, + snapshot.healthQueued ?? 0, + snapshot.normalQueued, + snapshot.backgroundQueued, + ].some((count) => count > 0); +} + +/** + * Split sweep failures into transient and terminal, because the two need + * opposite scheduling. A scheduler rejection is the store's own lane reporting + * saturation — the same condition {@link hasActiveStorePressure} samples, only + * raised as a throw because the load arrived after the gate rather than before + * it. Those must stay on the short retry path or the backlog strands until the + * periodic backstop. + * + * Everything else is terminal until proven otherwise and must NOT be retried on + * that cadence: a genuinely broken sweep would spin every few seconds forever, + * trading a stranded backlog for a hot loop. Widening this predicate therefore + * needs the same scrutiny as adding a retry. + */ +function isTransientStorePressure(error: unknown): boolean { + return error instanceof StoreSchedulerBusyError; +} + +export interface FinalizedSwmCleanupServiceOptions { + store: TripleStore; + writeLocks?: Map>; + eventBus?: EventBus; + /** + * Both enumerations are handed the sweep's background query options, so an + * implementation that reaches the store keeps discovery off the foreground + * lane: a cold or past-revalidation graph-set index resolves its refresh + * priority from the caller (see GraphSetIndexStore.refreshPriority), where an + * absent priority defaults to `normal`. An implementation backed by a shared + * cache may ignore them — see the production wiring for why. + */ + listContextGraphIds: (options: QueryOptions) => Promise; + listSharedMemoryMetaGraphs: (contextGraphId: string, options: QueryOptions) => Promise; + now?: () => number; + maxCandidatesPerSweep?: number; + wallClockBudgetMs?: number; +} + +/** + * The sole owner of finalized-SWM discovery, payload verification and deletion. + * Finalization and synchronization may create/re-arm durable tasks, but neither + * path calls this service or waits for its store work. + */ +export class FinalizedSwmCleanupService { + private readonly store: TripleStore; + private readonly writeLocks: Map> | undefined; + private readonly eventBus: EventBus | undefined; + private readonly listContextGraphIds: (options: QueryOptions) => Promise; + private readonly listSharedMemoryMetaGraphs: ( + contextGraphId: string, + options: QueryOptions, + ) => Promise; + private readonly now: () => number; + private readonly maxCandidatesPerSweep: number; + private readonly wallClockBudgetMs: number; + private readonly graphWriteGen: GraphWriteGenSource | null; + private readonly log = new Logger('FinalizedSwmCleanupService'); + /** Whole-node totals from the last rotation that closed; never a partial sum. */ + private lastKnownBacklogDepth = 0; + private lastKnownOldestMarkerAt: number | null = null; + /** + * Context graphs still owed a visit in the rotation currently in progress, or + * `null` when none is. Entry 0 doubles as the resumption cursor: a sweep that + * yields on pressure or wall clock persists its remaining tail here, so the + * next sweep continues instead of re-walking the same prefix forever and + * never reaching markers in later context graphs. + */ + private rotationPending: string[] | null = null; + private rotationBacklogDepth = 0; + private rotationOldestMarkerAt: number | null = null; + /** + * Head of the rotation the previous sweep started from, and how many + * consecutive sweeps have started from it without completing it. A context + * graph whose enumeration cannot finish inside one slice would otherwise hold + * the cursor forever and starve every context graph behind it — the same + * "later context graphs are never reached" failure the cursor exists to fix. + */ + private rotationHeadContextGraphId: string | null = null; + private rotationHeadStalledSweeps = 0; + /** + * Where to begin the next slice's walk of one context graph's meta graphs. + * A graph whose meta graphs cannot all be measured inside one slice would + * otherwise restart at index 0 every time and yield at the same point, so + * everything past that point is never cleaned — the rotation cursor's own + * failure mode, one level down. Rotating the start covers them all across + * successive slices. + */ + private metaGraphResumeCursor: { contextGraphId: string; offset: number } | null = null; + /** + * Keyset position within the one meta graph whose candidate page is unfinished. + * Discovery is `ORDER BY ?task LIMIT n`, so without this the same prefix is + * re-selected every sweep and its successors are never examined — the third + * instance of the cursor-less re-walk this service has, after the context-graph + * rotation and the meta-graph walk. Tasks that can never be acted on (a head + * that no longer verifies) otherwise hold the whole node-wide deletion budget + * indefinitely. + * + * A single entry suffices, and is bounded by construction rather than by an + * eviction policy: charging `remaining` per examined candidate means a full page + * always spends the rest of the budget, so no other meta graph runs discovery + * in that slice, and a short page clears the cursor. At most one meta graph can + * hold an unfinished page at a time. + */ + private taskCursor: { swmMetaGraph: string; afterTaskSubject: string } | null = null; + + constructor(options: FinalizedSwmCleanupServiceOptions) { + this.store = options.store; + this.writeLocks = options.writeLocks; + this.eventBus = options.eventBus; + this.listContextGraphIds = options.listContextGraphIds; + this.listSharedMemoryMetaGraphs = options.listSharedMemoryMetaGraphs; + this.now = options.now ?? Date.now; + this.maxCandidatesPerSweep = Math.min( + 16, + Math.max(1, Math.floor(options.maxCandidatesPerSweep ?? 4)), + ); + this.wallClockBudgetMs = Math.max(1, Math.floor(options.wallClockBudgetMs ?? 10_000)); + this.graphWriteGen = asGraphWriteGenSource(options.store); + } + + /** Run one bounded, idle-only maintenance slice. */ + async runSweep(): Promise { + // A deferred sweep republishes the last rotation's whole-node totals rather + // than a partial sum, and marks them stale so the SLO surface can tell + // "deferred under load" apart from "nothing to do" — the distinction an + // operator needs while only `pressureSkips` is moving. + const deferredResult = ( + deletedItems: number, + reason: 'pressure' | 'budget', + ): FinalizedSwmCleanupSweepResult => ({ + backlogDepth: this.lastKnownBacklogDepth, + oldestMarkerAt: this.lastKnownOldestMarkerAt, + deletedItems, + pressureSkipped: reason === 'pressure', + ...(reason === 'budget' ? { budgetExhausted: true } : {}), + stale: true, + }); + const underPressure = () => hasActiveStorePressure(this.store.getPressureSnapshot?.()); + if (underPressure()) return deferredResult(0, 'pressure'); + + const deadline = this.now() + this.wallClockBudgetMs; + const deadlineSignal = AbortSignal.timeout(this.wallClockBudgetMs); + // The signal bounds enumeration by wall clock, which nothing did before. + // It is not load-responsive preemption: pressure is sampled at boundaries + // only, and the background lane bounds concurrency rather than interrupting + // a query already running. Cutting an in-flight scan short when load + // arrives mid-enumeration remains an accepted gap. + const discoveryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup.discover', + signal: deadlineSignal, + }; + let remaining = this.maxCandidatesPerSweep; + let deletedItems = 0; + + // Pressure is checked before every discovery boundary, including the first + // potentially expensive context-graph enumeration. + if (underPressure()) return deferredResult(0, 'pressure'); + let contextGraphIds: string[]; + try { + contextGraphIds = await this.listContextGraphIds(discoveryOptions); + } catch (error) { + if (deadlineSignal.aborted) return deferredResult(0, 'budget'); + // A scheduler rejection IS store pressure — the background lane reporting + // that it is saturated — it just arrives as a throw instead of through the + // point-in-time snapshot gate, which cannot see load that lands between the + // gate and the query. Classifying it keeps the sweep on the retry path and + // counted in `pressureSkips`; letting it escape strands the backlog until + // the periodic backstop for a transient, retryable reason. Every other + // error still throws: an unknown fault must not hot-retry every few + // seconds, and the backstop is the right cadence for it. + if (isTransientStorePressure(error)) return deferredResult(0, 'pressure'); + throw error; + } + const pending = this.resumeRotation(contextGraphIds); + + let index = 0; + try { + for (; index < pending.length; index += 1) { + const contextGraphId = pending[index]!; + // Yielding keeps the unvisited tail, so the cursor never advances past a + // context graph this sweep did not finish measuring. + let metaGraphStart = 0; + let metaGraphsMeasured = 0; + const yieldRotation = (reason: 'pressure' | 'budget'): FinalizedSwmCleanupSweepResult => { + this.rotationPending = pending.slice(index); + // Resume the meta-graph walk past what this slice already measured, so + // a context graph too large for one slice still covers all of them. + if (metaGraphsMeasured > 0) { + this.metaGraphResumeCursor = { + contextGraphId, + offset: metaGraphStart + metaGraphsMeasured, + }; + } + return deferredResult(deletedItems, reason); + }; + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + let metaGraphs: string[]; + try { + metaGraphs = await this.listSharedMemoryMetaGraphs(contextGraphId, discoveryOptions); + } catch (error) { + if (deadlineSignal.aborted) return yieldRotation('budget'); + if (isTransientStorePressure(error)) return yieldRotation('pressure'); + throw error; + } + // Resume where the last slice stopped inside this context graph, wrapping, + // so a graph too large for one slice still reaches every meta graph rather + // than re-walking the same prefix. The walk still covers all of them, so + // the contribution below stays a whole-graph sum. + if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId && metaGraphs.length > 0) { + metaGraphStart = this.metaGraphResumeCursor.offset % metaGraphs.length; + if (metaGraphStart > 0) { + metaGraphs = [...metaGraphs.slice(metaGraphStart), ...metaGraphs.slice(0, metaGraphStart)]; + } + } + // A context graph contributes to the rotation total only once every one of + // its meta graphs has been measured. Yielding part-way discards the partial + // sum, so the re-measure on resume cannot double-count it. + let contextGraphBacklogDepth = 0; + let contextGraphOldestMarkerAt: number | null = null; + for (const swmMetaGraph of metaGraphs) { + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + if (remaining > 0) { + let cleanup: Awaited>; + try { + cleanup = await this.cleanupMetaGraph({ + contextGraphId, + swmMetaGraph, + maxCandidates: remaining, + signal: deadlineSignal, + ...(this.taskCursor?.swmMetaGraph === swmMetaGraph + ? { afterTaskSubject: this.taskCursor.afterTaskSubject } + : {}), + }); + } catch (error) { + if (deadlineSignal.aborted) return yieldRotation('budget'); + if (isTransientStorePressure(error)) return yieldRotation('pressure'); + throw error; + } + deletedItems += cleanup.deletedItems; + remaining -= cleanup.examinedCandidates; + // Advance while the page is full; wrap when it is short so a stuck + // prefix is skipped without stranding the suffix behind it. + this.taskCursor = cleanup.pageWasFull && cleanup.lastTaskSubject !== undefined + ? { swmMetaGraph, afterTaskSubject: cleanup.lastTaskSubject } + : null; + } + if (underPressure()) return yieldRotation('pressure'); + if (this.now() >= deadline) return yieldRotation('budget'); + // Deliberately not gated on `remaining`: that budget bounds deletion + // work (head resolution, VM/SWM verification, lock, atomic replace), + // while this is one cheap background aggregate and the sole source of + // the backlog metric. Capping the metric with the deletion budget would + // make it under-report exactly when the backlog is deepest. + let backlog: { depth: number; oldestMarkerAt: number | null }; + try { + backlog = await this.inspectBacklog(swmMetaGraph, deadlineSignal); + } catch (error) { + if (deadlineSignal.aborted) return yieldRotation('budget'); + // Also covers this method's own defensive pressure throw, which the + // snapshot gate above races with rather than prevents. + if (isTransientStorePressure(error)) return yieldRotation('pressure'); + throw error; + } + contextGraphBacklogDepth += backlog.depth; + if ( + backlog.oldestMarkerAt !== null + && ( + contextGraphOldestMarkerAt === null + || backlog.oldestMarkerAt < contextGraphOldestMarkerAt + ) + ) { + contextGraphOldestMarkerAt = backlog.oldestMarkerAt; + } + metaGraphsMeasured += 1; + await new Promise((resolve) => setImmediate(resolve)); + } + // Every meta graph was measured, so the next visit starts from the top. + if (this.metaGraphResumeCursor?.contextGraphId === contextGraphId) { + this.metaGraphResumeCursor = null; + } + this.rotationBacklogDepth += contextGraphBacklogDepth; + if ( + contextGraphOldestMarkerAt !== null + && ( + this.rotationOldestMarkerAt === null + || contextGraphOldestMarkerAt < this.rotationOldestMarkerAt + ) + ) { + this.rotationOldestMarkerAt = contextGraphOldestMarkerAt; + } + } + } catch (error) { + // Every exit from this loop must persist the cursor, including a terminal + // fault. Context graphs completed earlier in this sweep are already + // credited to `rotationBacklogDepth`, and `resumeRotation` only zeroes the + // accumulator when `rotationPending` is null — which a throw never + // produces. Leaving the cursor stale makes the next sweep resume from the + // old tail, re-measure those graphs and add them a second time, and the + // inflated sum eventually publishes as `stale: false`: presented as a + // trustworthy fresh whole-node measurement. Catching around the whole loop + // rather than at each throw site means a future raise site cannot miss it. + this.rotationPending = pending.slice(index); + throw error; + } + + // The rotation closed: every context graph was measured exactly once, so the + // accumulated sums are a whole-node total again. Reached with an empty tail + // too, when every context graph still owed a visit disappeared mid-rotation + // (or the node has none at all) — the total then covers exactly the graphs + // that still exist, which is the whole-node answer. + this.lastKnownBacklogDepth = this.rotationBacklogDepth; + this.lastKnownOldestMarkerAt = this.rotationOldestMarkerAt; + this.rotationPending = null; + this.rotationBacklogDepth = 0; + this.rotationOldestMarkerAt = null; + this.rotationHeadContextGraphId = null; + this.rotationHeadStalledSweeps = 0; + return { + backlogDepth: this.lastKnownBacklogDepth, + oldestMarkerAt: this.lastKnownOldestMarkerAt, + deletedItems, + pressureSkipped: false, + stale: false, + }; + } + + /** + * Continue the rotation in progress, dropping context graphs that disappeared + * since it started, or open a fresh one. Context graphs created mid-rotation + * join the next rotation rather than this one, so a closing total covers the + * set as of rotation start; the lag is bounded by one rotation. That lag is + * deliberately not reported as `stale` — on a node with any context-graph + * churn the flag would be true almost always, destroying the deferred-versus- + * drained signal it exists to carry. + * + * Guarantees forward progress: after two consecutive slices fail to finish the + * head, the third either defers it to the back of the rotation, so the context + * graphs behind it are still served, or — when it is the only one left, where + * deferring is a no-op — closes the rotation and re-seeds, so the rest of the + * node is swept again. Either way the stalled graph keeps its place and is + * retried; neither path drops it. + */ + private resumeRotation(contextGraphIds: string[]): string[] { + let pending: string[]; + if (this.rotationPending === null) { + this.rotationBacklogDepth = 0; + this.rotationOldestMarkerAt = null; + pending = [...contextGraphIds]; + } else { + const known = new Set(contextGraphIds); + pending = this.rotationPending.filter((contextGraphId) => known.has(contextGraphId)); + } + + const head = pending[0] ?? null; + if (head !== null && head === this.rotationHeadContextGraphId) { + this.rotationHeadStalledSweeps += 1; + } else { + this.rotationHeadContextGraphId = head; + this.rotationHeadStalledSweeps = 0; + } + if (this.rotationHeadStalledSweeps >= 2) { + const soleHead = pending.length === 1; + this.log.warn( + createOperationContext('system'), + `Context graph ${head} was not finished by ${this.rotationHeadStalledSweeps} ` + + `consecutive finalized-SWM cleanup slices; ` + + (soleHead + ? 'closing the rotation early so every other context graph is swept again' + : 'deferring it to the back of the rotation'), + ); + if (soleHead) { + // Rotating a one-element list is a no-op, so without this the rotation + // could never close: `rotationPending` would stay non-null forever, the + // re-seed below would never run, and every OTHER context graph on the + // node would never be swept again — #1996 reintroduced node-wide by the + // subsystem that exists to close it. + // + // The partial accumulator is discarded rather than published. An + // incomplete rotation must never become a whole-node total, so + // `lastKnownBacklogDepth` keeps the last complete measurement and every + // slice keeps reporting `stale` until a rotation genuinely finishes. + // A node whose budget cannot fit one context graph therefore reports an + // honestly unknown backlog, and this warning names the graph to raise it + // for. + this.rotationBacklogDepth = 0; + this.rotationOldestMarkerAt = null; + pending = [...contextGraphIds]; + } else { + pending = [...pending.slice(1), pending[0]!]; + } + this.rotationHeadContextGraphId = pending[0] ?? null; + this.rotationHeadStalledSweeps = 0; + } + return pending; + } + + /** Narrow test seam for one already-known SWM metadata graph. */ + async cleanupKnownMetaGraph(input: { + contextGraphId: string; + swmMetaGraph: string; + maxCandidates?: number; + signal?: AbortSignal; + }): Promise { + return (await this.cleanupMetaGraph(input)).deletedItems; + } + + private async inspectBacklog( + swmMetaGraph: string, + signal?: AbortSignal, + ): Promise<{ depth: number; oldestMarkerAt: number | null }> { + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + throw new StoreSchedulerBusyError( + 'queue_full', + 'background', + 'agent.finalizedSwmCleanup.backlog', + ); + } + const result = await this.store.query( + `SELECT (COUNT(DISTINCT ?task) AS ?count) (MIN(?markedAt) AS ?oldest) WHERE { + GRAPH <${assertSafeIri(swmMetaGraph)}> { + ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; + <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . + OPTIONAL { ?task <${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } + } + }`, + { + priority: 'background', + source: 'agent.finalizedSwmCleanup.backlog', + signal, + }, + ); + if (result.type !== 'bindings' || result.bindings.length === 0) { + return { depth: 0, oldestMarkerAt: null }; + } + const depth = Number.parseInt(stripOptionalLiteral(result.bindings[0]?.['count']) ?? '0', 10); + const oldestValue = stripOptionalLiteral(result.bindings[0]?.['oldest']); + const oldestMarkerAt = oldestValue ? Date.parse(oldestValue) : Number.NaN; + return { + depth: Number.isSafeInteger(depth) && depth > 0 ? depth : 0, + oldestMarkerAt: Number.isFinite(oldestMarkerAt) ? oldestMarkerAt : null, + }; + } + + private async cleanupMetaGraph(input: { + contextGraphId: string; + swmMetaGraph: string; + maxCandidates?: number; + signal?: AbortSignal; + /** + * Keyset position from a previous slice. Only `runSweep` supplies it, so the + * `cleanupKnownMetaGraph` seam keeps selecting from the top unchanged. + */ + afterTaskSubject?: string; + }): Promise<{ + deletedItems: number; + examinedCandidates: number; + lastTaskSubject?: string; + pageWasFull: boolean; + }> { + if (!this.writeLocks || hasActiveStorePressure(this.store.getPressureSnapshot?.())) { + return { deletedItems: 0, examinedCandidates: 0, pageWasFull: false }; + } + const limit = Math.min(16, Math.max(1, Math.floor(input.maxCandidates ?? 4))); + const queryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup.discover', + signal: input.signal, + }; + const result = await this.store.query( + `SELECT DISTINCT ?task ?ual ?version ?root ?shareId ?assertionGraph ?headFingerprint ?subGraphName WHERE { + GRAPH <${assertSafeIri(input.swmMetaGraph)}> { + ?task <${FINALIZED_SWM_CLEANUP_TASK_TYPE}> ; + <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root ; + <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ?headFingerprint ; + ?ual ; + ?version ; + ?shareId ; + ?assertionGraph . + OPTIONAL { ?task ?subGraphName } + ${input.afterTaskSubject === undefined + ? '' + : `FILTER(STR(?task) > ${JSON.stringify(input.afterTaskSubject)})`} + } + } ORDER BY ?task LIMIT ${limit}`, + queryOptions, + ); + if (result.type !== 'bindings') { + return { deletedItems: 0, examinedCandidates: 0, pageWasFull: false }; + } + + let cleared = 0; + for (const row of result.bindings) { + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) break; + const taskSubject = row['task']; + const ual = row['ual']; + const rawVersion = stripOptionalLiteral(row['version']); + const rawRoot = stripOptionalLiteral(row['root']); + const assertionGraph = row['assertionGraph']; + const expectedHeadFingerprint = stripOptionalLiteral(row['headFingerprint']); + const shareOperationId = stripOptionalLiteral(row['shareId']); + const subGraphName = stripOptionalLiteral(row['subGraphName']); + if ( + !taskSubject + || !taskSubject.startsWith('urn:dkg:finalized-swm-cleanup:') + || !ual + || !rawVersion + || !/^\d+$/.test(rawVersion) + || !rawRoot + || !assertionGraph + || !expectedHeadFingerprint + || !shareOperationId + ) continue; + + let scope: ReturnType; + let expectedMerkleRoot: Uint8Array; + try { + scope = createGraphKnowledgeAssetScope(ual, BigInt(rawVersion)); + expectedMerkleRoot = ethers.getBytes(rawRoot); + } catch { + continue; + } + if (scope.ual !== ual || expectedMerkleRoot.length !== 32) continue; + + let expectedHead: KnowledgeAssetWorkspaceHead | undefined; + try { + expectedHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager: new GraphManager(this.store), + contextGraphId: input.contextGraphId, + kaUal: scope.ual, + subGraphName, + queryOptions, + }); + } catch (error) { + if (error instanceof StoreSchedulerBusyError) break; + continue; + } + if ( + !expectedHead + || expectedHead.assertionVersion !== scope.assertionVersion + || expectedHead.shareOperationId !== shareOperationId + || expectedHead.assertionGraph !== assertionGraph + ) { + await this.retireStaleTask({ + contextGraphId: input.contextGraphId, + swmMetaGraph: input.swmMetaGraph, + taskSubject, + scope, + assertionGraph, + shareOperationId, + subGraphName, + queryOptions, + expectedHeadWasAbsent: !expectedHead, + }); + continue; + } + let privateMerkleRoot: Uint8Array | undefined; + try { + privateMerkleRoot = expectedHead.privateMerkleRoot + ? ethers.getBytes(expectedHead.privateMerkleRoot) + : undefined; + } catch { + continue; + } + const outcome = await this.clearIfStillExact({ + contextGraphId: input.contextGraphId, + scope, + taskSubject, + expectedHeadFingerprint, + expectedHead, + expectedMerkleRoot, + privateMerkleRoot, + subGraphName, + signal: input.signal, + }); + if (outcome === 'cleared') cleared += 1; + } + // `ORDER BY ?task` makes the last row the keyset position. A short page means + // the ordered set is exhausted, so the caller wraps instead of advancing. + // This is an optimisation, NOT a safety property: an empty page carries no + // last subject and therefore clears the cursor anyway, so dropping the + // short-page wrap costs one wasted empty query per cycle rather than + // stranding the suffix. Established by mutation — a wrap-only mutant is not + // killed, and this fallback is why. + const lastRow = result.bindings[result.bindings.length - 1]; + const lastTaskSubject = typeof lastRow?.['task'] === 'string' ? lastRow['task'] : undefined; + return { + deletedItems: cleared, + examinedCandidates: result.bindings.length, + ...(lastTaskSubject === undefined ? {} : { lastTaskSubject }), + pageWasFull: result.bindings.length >= limit, + }; + } + + private async retireStaleTask(input: { + contextGraphId: string; + swmMetaGraph: string; + taskSubject: string; + scope: ReturnType; + assertionGraph: string; + shareOperationId: string; + subGraphName?: string; + queryOptions: QueryOptions; + expectedHeadWasAbsent: boolean; + }): Promise { + if (!this.writeLocks) return; + const writePrefix = `${contextGraphDataUri(input.contextGraphId)}/`; + const preflightWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + let headlessPayloadAbsent = false; + if (input.expectedHeadWasAbsent) { + if (preflightWriteGen === undefined) return; + const payloadPresent = await this.store.query( + `ASK { GRAPH <${assertSafeIri(input.assertionGraph)}> { ?s ?p ?o } }`, + { + ...input.queryOptions, + source: 'agent.finalizedSwmCleanup.checkHeadlessPayload', + }, + ); + if (payloadPresent.type !== 'boolean' || payloadPresent.value) return; + headlessPayloadAbsent = true; + } + const lockKey = swmKaWriteLockKey( + input.contextGraphId, + input.subGraphName, + input.scope.ual, + ); + await withKeyedLocks(this.writeLocks, [lockKey], async () => { + if ( + preflightWriteGen !== undefined + && this.graphWriteGen?.getWriteGen(writePrefix) !== preflightWriteGen + ) return; + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager: new GraphManager(this.store), + contextGraphId: input.contextGraphId, + kaUal: input.scope.ual, + subGraphName: input.subGraphName, + queryOptions: input.queryOptions, + }); + } catch (error) { + if (error instanceof KnowledgeAssetWorkspaceHeadCorruptError) return; + throw error; + } + if ( + currentHead + && currentHead.assertionVersion === input.scope.assertionVersion + && currentHead.shareOperationId === input.shareOperationId + && currentHead.assertionGraph === input.assertionGraph + ) return; + if (!currentHead && !headlessPayloadAbsent) return; + + await this.store.deleteByPattern( + { graph: input.swmMetaGraph, subject: input.taskSubject }, + { + ...input.queryOptions, + source: currentHead + ? 'agent.finalizedSwmCleanup.retireSupersededTask' + : 'agent.finalizedSwmCleanup.retireAbsentTask', + }, + ); + }); + } + + private async clearIfStillExact(input: { + contextGraphId: string; + scope: ReturnType; + taskSubject: string; + expectedHeadFingerprint: string; + expectedHead: KnowledgeAssetWorkspaceHead; + expectedMerkleRoot: Uint8Array; + privateMerkleRoot?: Uint8Array; + subGraphName?: string; + signal?: AbortSignal; + }): Promise<'cleared' | 'absent' | 'preserved'> { + if (!this.writeLocks) return 'preserved'; + const graphManager = new GraphManager(this.store); + const metaGraph = graphManager.sharedMemoryMetaUri(input.contextGraphId, input.subGraphName); + const headSubject = workspaceKnowledgeAssetHeadSubject(input.scope.ual); + const cleanupRootObject = JSON.stringify( + ethers.hexlify(input.expectedMerkleRoot).toLowerCase(), + ); + const verificationQueryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup', + signal: input.signal, + }; + const commitQueryOptions: QueryOptions = { + priority: 'background', + source: 'agent.finalizedSwmCleanup', + }; + if ( + finalizedSwmCleanupHeadFingerprint(input.expectedHead) + !== input.expectedHeadFingerprint + ) return 'preserved'; + + const writePrefix = `${contextGraphDataUri(input.contextGraphId)}/`; + const preflightWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) return 'preserved'; + const vmVerification = await verifyExactGraphScopedLayer({ + store: this.store, + contextGraphId: input.contextGraphId, + scope: input.scope, + layer: MemoryLayer.VerifiableMemory, + publicTripleCount: input.expectedHead.publicTripleCount, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + expectedPublicQuadsDigest: input.expectedHead.publicQuadsDigest, + subGraphName: input.subGraphName, + queryOptions: verificationQueryOptions, + }); + if (vmVerification.status !== 'verified') { + this.log.warn( + createOperationContext('system'), + `Preserving ${input.scope.ual}; VM no longer matches cleanup task (${vmVerification.status})`, + ); + return 'preserved'; + } + if (hasActiveStorePressure(this.store.getPressureSnapshot?.())) return 'preserved'; + const swmVerification = await verifyExactGraphScopedLayer({ + store: this.store, + contextGraphId: input.contextGraphId, + scope: input.scope, + layer: MemoryLayer.SharedWorkingMemory, + publicTripleCount: input.expectedHead.publicTripleCount, + privateMerkleRoot: input.privateMerkleRoot, + expectedMerkleRoot: input.expectedMerkleRoot, + expectedPublicQuadsDigest: input.expectedHead.publicQuadsDigest, + subGraphName: input.subGraphName, + queryOptions: verificationQueryOptions, + }); + if ( + swmVerification.status !== 'verified' + && !(swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) + ) { + this.log.warn( + createOperationContext('system'), + `Preserving ${input.scope.ual}; SWM no longer matches cleanup task (${swmVerification.status})`, + ); + return 'preserved'; + } + const verifiedWriteGen = this.graphWriteGen?.getWriteGen(writePrefix); + if (preflightWriteGen !== undefined && verifiedWriteGen !== preflightWriteGen) { + return 'preserved'; + } + + const lockKey = swmKaWriteLockKey( + input.contextGraphId, + input.subGraphName, + input.scope.ual, + ); + const outcome = await withKeyedLocks(this.writeLocks, [lockKey], async () => { + if ( + verifiedWriteGen !== undefined + && this.graphWriteGen?.getWriteGen(writePrefix) !== verifiedWriteGen + ) return 'preserved' as const; + + let currentHead: KnowledgeAssetWorkspaceHead | undefined; + try { + currentHead = await resolveKnowledgeAssetWorkspaceHead({ + store: this.store, + graphManager, + contextGraphId: input.contextGraphId, + kaUal: input.scope.ual, + subGraphName: input.subGraphName, + queryOptions: commitQueryOptions, + }); + } catch (error) { + if (!(error instanceof KnowledgeAssetWorkspaceHeadCorruptError)) throw error; + return 'preserved' as const; + } + if (!currentHead) { + if (swmVerification.status === 'count-mismatch' && swmVerification.actualCount === 0) { + await this.store.deleteByPattern( + { graph: metaGraph, subject: input.taskSubject }, + { ...commitQueryOptions, source: 'agent.finalizedSwmCleanup.retireAbsentTask' }, + ); + return 'absent' as const; + } + return 'preserved' as const; + } + if ( + !sameKnowledgeAssetWorkspaceHead(currentHead, input.expectedHead) + || finalizedSwmCleanupHeadFingerprint(currentHead) !== input.expectedHeadFingerprint + ) return 'preserved' as const; + + const marker = await this.store.query( + `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` + + `<${assertSafeIri(input.taskSubject)}> <${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ` + + `${cleanupRootObject} ; <${FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE}> ` + + `${JSON.stringify(input.expectedHeadFingerprint)} } }`, + commitQueryOptions, + ); + if (marker.type !== 'boolean' || !marker.value) return 'preserved' as const; + + const replaced = await tryReplaceGraphAndSubjectAtomically( + this.store, + swmVerification.graphUri, + [], + metaGraph, + headSubject, + [], + commitQueryOptions, + ); + if (!replaced) { + throw Object.assign( + new Error('Finalized SWM cleanup requires atomic graph-and-head replacement support'), + { code: 'SWM_ATOMIC_CLEANUP_UNSUPPORTED' }, + ); + } + await this.store.deleteByPattern( + { graph: metaGraph, subject: input.taskSubject }, + { ...commitQueryOptions, source: 'agent.finalizedSwmCleanup.retireTask' }, + ); + return swmVerification.status === 'verified' ? 'cleared' as const : 'absent' as const; + }); + + if (outcome === 'cleared') { + this.eventBus?.emit(DKGEvent.MEMORY_GRAPH_CHANGED, { + contextGraphId: input.contextGraphId, + layers: ['swm'], + subGraphName: input.subGraphName, + operation: 'shared_working_memory_finalized', + source: 'background-cleanup', + counts: { triples: input.expectedHead.publicTripleCount }, + }); + this.log.info( + createOperationContext('system'), + `Cleared finalized graph-scoped SWM assertion ${input.scope.ual}`, + ); + } + return outcome; + } +} diff --git a/packages/agent/src/finalized-swm-cleanup-worker.ts b/packages/agent/src/finalized-swm-cleanup-worker.ts new file mode 100644 index 0000000000..3671094714 --- /dev/null +++ b/packages/agent/src/finalized-swm-cleanup-worker.ts @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Independent, pressure-gated maintenance worker for finalized SWM cleanup. + * + * Foreground finalization and catch-up only call {@link wake}; they never await + * discovery, payload verification, or deletion. The worker owns retry and + * single-flight semantics so repeated wake-ups cannot create overlapping store + * scans. A periodic caller remains the restart/lost-wakeup safety net. + */ + +export interface FinalizedSwmCleanupSweepResult { + /** + * Cleanup tasks still requiring a safe idle pass, as a whole-node total from + * the last context-graph rotation that ran to completion. Never a partial sum: + * a sweep that only covers part of a rotation republishes the previous total + * and sets {@link stale}. + */ + backlogDepth: number; + /** Oldest surviving cleanup marker, or null when the backlog is empty/unknown. */ + oldestMarkerAt: number | null; + /** Exact SWM lifecycles deleted by this sweep. */ + deletedItems: number; + /** True when pressure appeared before or during discovery/cleanup. */ + pressureSkipped: boolean; + /** True when the bounded slice yielded with more discovery work possible. */ + budgetExhausted?: boolean; + /** + * True when `backlogDepth`/`oldestMarkerAt` are not a current measurement — + * this sweep deferred on pressure or wall clock, or is part-way through a + * rotation. Distinguishes a deferred backlog from an empty one. + */ + stale?: boolean; +} + +export interface FinalizedSwmCleanupStats { + backlogDepth: number; + oldestMarkerAgeMs: number | null; + /** + * True until a full context-graph rotation has published a whole-node + * measurement, and again whenever the latest sweep deferred. While true, + * `backlogDepth` 0 / `oldestMarkerAgeMs` null mean "unknown", not "empty". + */ + backlogStale: boolean; + pressureSkips: number; + deletedItems: number; + runs: number; + lastRunAt: string | null; + lastError: string | null; +} + +export interface FinalizedSwmCleanupWorkerOptions { + sweep: () => Promise; + now?: () => number; + retryDelayMs?: number; + setTimer?: (fn: () => void, delayMs: number) => ReturnType; + clearTimer?: (timer: ReturnType) => void; + onError?: (error: unknown) => void; +} + +export class FinalizedSwmCleanupWorker { + private readonly sweep: () => Promise; + private readonly now: () => number; + private readonly retryDelayMs: number; + private readonly setTimer: NonNullable; + private readonly clearTimer: NonNullable; + private readonly onError: (error: unknown) => void; + private wakeTimer: ReturnType | null = null; + private inFlight: Promise | null = null; + private wakePendingDelayMs: number | null = null; + private closed = false; + private oldestMarkerAt: number | null = null; + private readonly stats: FinalizedSwmCleanupStats = { + backlogDepth: 0, + oldestMarkerAgeMs: null, + // Nothing has been measured yet, which is not the same as an empty backlog. + backlogStale: true, + pressureSkips: 0, + deletedItems: 0, + runs: 0, + lastRunAt: null, + lastError: null, + }; + + constructor(options: FinalizedSwmCleanupWorkerOptions) { + this.sweep = options.sweep; + this.now = options.now ?? Date.now; + this.retryDelayMs = Math.max(1, Math.floor(options.retryDelayMs ?? 5_000)); + this.setTimer = options.setTimer ?? ((fn, delayMs) => setTimeout(fn, delayMs)); + this.clearTimer = options.clearTimer ?? clearTimeout; + this.onError = options.onError ?? (() => {}); + } + + /** Schedule an independent maintenance pass; never waits for store work. */ + wake(delayMs = 0): void { + if (this.closed) return; + if (this.inFlight) { + const normalizedDelay = Math.max(0, Math.floor(delayMs)); + this.wakePendingDelayMs = this.wakePendingDelayMs === null + ? normalizedDelay + : Math.min(this.wakePendingDelayMs, normalizedDelay); + return; + } + if (this.wakeTimer) return; + this.wakeTimer = this.setTimer(() => { + this.wakeTimer = null; + void this.run(); + }, Math.max(0, Math.floor(delayMs))); + this.wakeTimer.unref?.(); + } + + /** Test/diagnostic seam that joins the current pass without starting overlap. */ + async runNow(): Promise { + if (this.closed) return; + if (this.wakeTimer) { + this.clearTimer(this.wakeTimer); + this.wakeTimer = null; + } + await this.run(); + } + + snapshot(): FinalizedSwmCleanupStats { + return { + ...this.stats, + oldestMarkerAgeMs: this.oldestMarkerAt === null + ? null + : Math.max(0, this.now() - this.oldestMarkerAt), + }; + } + + async close(): Promise { + this.closed = true; + this.wakePendingDelayMs = null; + if (this.wakeTimer) { + this.clearTimer(this.wakeTimer); + this.wakeTimer = null; + } + await this.inFlight; + } + + private async run(): Promise { + if (this.closed) return; + if (this.inFlight) { + this.wakePendingDelayMs = 0; + await this.inFlight; + return; + } + const task = this.performSweep(); + this.inFlight = task; + try { + await task; + } finally { + if (this.inFlight === task) this.inFlight = null; + if (this.closed) return; + if (this.wakePendingDelayMs !== null) { + const delayMs = this.wakePendingDelayMs; + this.wakePendingDelayMs = null; + this.wake(delayMs); + } + } + } + + private async performSweep(): Promise { + const startedAt = this.now(); + this.stats.runs += 1; + this.stats.lastRunAt = new Date(startedAt).toISOString(); + try { + const result = await this.sweep(); + this.stats.backlogDepth = Math.max(0, Math.floor(result.backlogDepth)); + this.stats.backlogStale = result.stale === true; + this.oldestMarkerAt = result.oldestMarkerAt; + this.stats.oldestMarkerAgeMs = this.oldestMarkerAt === null + ? null + : Math.max(0, this.now() - this.oldestMarkerAt); + this.stats.deletedItems += Math.max(0, Math.floor(result.deletedItems)); + if (result.pressureSkipped) this.stats.pressureSkips += 1; + this.stats.lastError = null; + + // Pressure is expected and retryable. Continue a productive bounded + // drain without waiting for the 15-minute periodic backstop; a stuck or + // fail-closed backlog (no deletion) waits for the next explicit wake. + if ( + result.pressureSkipped + || result.budgetExhausted === true + || (result.deletedItems > 0 && result.backlogDepth > 0) + ) { + this.wake(this.retryDelayMs); + } + } catch (error) { + this.stats.lastError = error instanceof Error ? error.message : String(error); + this.onError(error); + } + } +} diff --git a/packages/agent/src/graph-scoped-layer-verification.ts b/packages/agent/src/graph-scoped-layer-verification.ts new file mode 100644 index 0000000000..42adbf2677 --- /dev/null +++ b/packages/agent/src/graph-scoped-layer-verification.ts @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { + MemoryLayer, + assertSafeIri, + createGraphKnowledgeAssetScope, + knowledgeAssetLayerGraphUri, +} from '@origintrail-official/dkg-core'; +import type { Quad, QueryOptions, TripleStore } from '@origintrail-official/dkg-storage'; +import { + computeFlatKCRootV10 as computeFlatKCRoot, + type KnowledgeAssetWorkspaceHead, + workspacePublicQuadsDigest, +} from '@origintrail-official/dkg-publisher'; + +export type ExactGraphScopedLayerVerification = + | { + status: 'verified'; + graphUri: string; + quads: Quad[]; + merkleRoot: Uint8Array; + } + | { + status: 'count-mismatch'; + graphUri: string; + actualCount: number; + } + | { + status: 'merkle-mismatch'; + graphUri: string; + } + | { + status: 'head-mismatch'; + graphUri: string; + }; + +function equalBytes(left: Uint8Array, right: Uint8Array): boolean { + return left.length === right.length + && left.every((byte, index) => byte === right[index]); +} + +/** + * Verify one exact graph-scoped VM or SWM layer from live payload content. + * This helper deliberately has no cleanup policy and is shared by foreground + * finalization and the independent finalized-SWM maintenance component. + */ +export async function verifyExactGraphScopedLayer(input: { + store: TripleStore; + contextGraphId: string; + scope: ReturnType; + layer: MemoryLayer.SharedWorkingMemory | MemoryLayer.VerifiableMemory; + publicTripleCount: number; + privateMerkleRoot?: Uint8Array; + expectedMerkleRoot: Uint8Array; + expectedPublicQuadsDigest?: KnowledgeAssetWorkspaceHead['publicQuadsDigest']; + subGraphName?: string; + queryOptions?: QueryOptions; +}): Promise { + const graphUri = knowledgeAssetLayerGraphUri( + input.contextGraphId, + input.layer, + input.scope, + input.subGraphName, + ); + const result = await input.store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(graphUri)}> { ?s ?p ?o } }`, + input.queryOptions, + ); + const quads = result.type === 'quads' + ? result.quads.map((quad) => ({ ...quad, graph: '' })) + : []; + if (quads.length !== input.publicTripleCount) { + return { status: 'count-mismatch', graphUri, actualCount: quads.length }; + } + const merkleRoot = computeFlatKCRoot( + quads, + input.privateMerkleRoot ? [input.privateMerkleRoot] : [], + ); + if (!equalBytes(merkleRoot, input.expectedMerkleRoot)) { + return { status: 'merkle-mismatch', graphUri }; + } + if ( + input.expectedPublicQuadsDigest !== undefined + && workspacePublicQuadsDigest(quads) !== input.expectedPublicQuadsDigest + ) { + return { status: 'head-mismatch', graphUri }; + } + return { status: 'verified', graphUri, quads, merkleRoot }; +} diff --git a/packages/agent/src/sync/graph-scoped-swm-recovery.ts b/packages/agent/src/sync/graph-scoped-swm-recovery.ts index 2434ac4a8c..374cf53f74 100644 --- a/packages/agent/src/sync/graph-scoped-swm-recovery.ts +++ b/packages/agent/src/sync/graph-scoped-swm-recovery.ts @@ -44,9 +44,13 @@ export interface GraphScopedSwmRecoveryDescriptor { readonly shareOperationId: string; readonly publicQuadsDigest: string; readonly publicQuadsCount: number; + readonly privateMerkleRoot?: string; + readonly privateTripleCount: number; readonly publicSnapshotRef?: string; readonly publicSnapshotGraph?: string; readonly publisherPeerId: string; + readonly accessPolicy?: 'public' | 'ownerOnly' | 'allowList'; + readonly allowedPeers: readonly string[]; readonly subGraphName?: string; /** Only the active head and its referenced operation, for snapshot fetch. */ readonly metadataQuads: readonly Quad[]; @@ -222,6 +226,15 @@ export function parseGraphScopedSwmRecoveryDescriptors(params: { if (!publisherPeerId) { throw new Error(`Graph-scoped SWM operation ${operationSubject} has an empty publisherPeerId`); } + const accessPolicy = optionalLiteral(operationRows, ACCESS_POLICY, 'accessPolicy')?.trim() as + | 'public' + | 'ownerOnly' + | 'allowList' + | undefined; + const allowedPeers = distinctObjects(operationRows, ALLOWED_PEER) + .map(stripLiteral) + .filter(Boolean) + .sort(); descriptors.push({ metaGraph, headSubject, @@ -232,9 +245,13 @@ export function parseGraphScopedSwmRecoveryDescriptors(params: { shareOperationId, publicQuadsDigest, publicQuadsCount, + ...(privateRoot ? { privateMerkleRoot: stripLiteral(privateRoot).toLowerCase() } : {}), + privateTripleCount, ...(publicSnapshotRef ? { publicSnapshotRef } : {}), ...(publicSnapshotGraph ? { publicSnapshotGraph } : {}), publisherPeerId, + ...(accessPolicy ? { accessPolicy } : {}), + allowedPeers, ...(subGraphName ? { subGraphName } : {}), metadataQuads: [...headRows, ...operationRows], }); diff --git a/packages/agent/src/sync/requester/shared-memory-sync.ts b/packages/agent/src/sync/requester/shared-memory-sync.ts index 9e0edf6d0d..0d039221e8 100644 --- a/packages/agent/src/sync/requester/shared-memory-sync.ts +++ b/packages/agent/src/sync/requester/shared-memory-sync.ts @@ -296,6 +296,9 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro let materializationFailures = 0; let materializedQuads = 0; const materializedKeys = new Set(); + const replacedGraphScopedMetaKeys = new Set(); + const quadKey = (quad: Quad): string => + `${quad.graph}\u0000${quad.subject}\u0000${quad.predicate}\u0000${quad.object}`; const materializeReadySnapshot = async (snapshotRef: string): Promise => { const descriptors = snapshotDescriptorsByRef.get(snapshotRef); if (!descriptors?.length || !snapshotMaterializer || !publicSnapshotStore) return; @@ -314,6 +317,15 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // re-checks below stop the other failure the lock alone cannot: // replacing newer content with an older verified snapshot. // + // Every path below must leave a constant-size GC task armed + // when an earlier finalized cleanup left an operation + // tombstone, or a resurrected SWM copy would never be + // collected. `replaceHeadMetadata` does that from the tombstone + // read it already performs, so only the paths that never reach + // it call `ensureFinalizedCleanupTask` explicitly. The + // independent GC still owns all discovery, graph verification + // and deletion; sync neither performs nor awaits that work. + // // (a) Version ordering. A stored head newer than the descriptor // means gossip advanced this KA past our snapshot; replacing // would be overwrite-with-older, byte-for-byte the regression @@ -327,6 +339,13 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro storedHead.version !== null && storedVersionOutranksDescriptor(storedHead.version, descriptor.assertionVersion) ) { + // Sync writes nothing here, but a previous round may have + // resurrected this KA and the GC may since have retired its + // task, so the tombstone still has to re-arm one. + await snapshotMaterializer.ensureFinalizedCleanupTask(pid, descriptor); + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); + } materializedKeys.add(graphKey); logDebug(ctx, `SWM sync for "${pid}": snapshot ${snapshotRef} superseded by ` + `stored version ${storedHead.version} (descriptor ${descriptor.assertionVersion}); skipping`); @@ -338,15 +357,24 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // digest is an OLDER version of the same size and must be // replaced, not skipped. if (await snapshotMaterializer.isGraphAssetMaterialized(descriptor)) { - if (storedHead.needsRepair) { + if (storedHead.version === null || storedHead.needsRepair) { // Content is already this descriptor's, but the head - // subject still carries union-insert residue (several - // version/operation rows) — e.g. a prior round replaced - // the graph and then failed before finishing the metadata - // swap. Collapse the head now; the fresh verified meta for - // this descriptor is re-inserted after the snapshot phase, - // exactly like the replace path below. + // is absent or still carries union-insert residue (several + // version/operation rows) — e.g. a prior round replaced the + // graph and then failed before finishing the metadata swap. + // Install the verified head while the writer lock is held. + // This also re-arms any finalized-cleanup task. await snapshotMaterializer.replaceHeadMetadata(pid, descriptor); + } else { + // Clean head, content already present: nothing is written, + // so re-arm the GC task on its own. + await snapshotMaterializer.ensureFinalizedCleanupTask(pid, descriptor); + } + // Never append graph-scoped metadata again after releasing + // this lock. For a clean head it is redundant; for a newer + // local lifecycle it would recreate stale union residue. + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); } materializedKeys.add(graphKey); return; @@ -361,11 +389,26 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro // Graph first, THEN the head swap — a crash between the two // leaves content newer than the head, which the next round // repairs (digest matches → head collapsed above). The swap - // deletes the old head + its operations so the append-style - // `storeInsert(processed.verifiedMeta)` below lands on a clean - // subject instead of stacking a second version onto it - // (LIMIT-1 head readers would otherwise see an arbitrary mix). + // deletes the old head + its operations, installs the verified + // replacement under this same writer lock, and excludes those + // rows from the append-style metadata insert below. Otherwise + // a live write could land between the swap and the append and + // leave an arbitrary multi-version head. + // + // The finalized-cleanup task is armed BY this swap, i.e. after + // the payload and head are both in place, never before the + // payload. Arming first was safe only because arm-and-write + // shared one hold of this writer lock, so the GC would block + // and then bail on a changed write generation + // (`retireStaleTask`). That ratchet is optional — it refuses to + // retire when the write-gen capability is absent — so ordering + // the arm after the write removes the dependency on it instead + // of relying on it, and leaves the task armed only once the + // head matches on every field `clearIfStillExact` checks. await snapshotMaterializer.replaceHeadMetadata(pid, descriptor); + for (const quad of descriptor.metadataQuads) { + replacedGraphScopedMetaKeys.add(quadKey(quad)); + } materializedKeys.add(graphKey); materializedGraphs += 1; materializedQuads += asset.quads.length; @@ -460,8 +503,13 @@ export async function runSharedMemorySync(context: SharedMemorySyncContext): Pro summary.insertedTriples += validWsQuads.length; summary.insertedDataTriples += validWsQuads.length; } + const remainingVerifiedMeta = processed.verifiedMeta.filter( + (quad) => !replacedGraphScopedMetaKeys.has(quadKey(quad)), + ); + if (remainingVerifiedMeta.length > 0) { + await storeInsert(remainingVerifiedMeta); + } if (processed.verifiedMeta.length > 0) { - await storeInsert(processed.verifiedMeta); summary.insertedTriples += processed.verifiedMeta.length; summary.insertedMetaTriples += processed.verifiedMeta.length; } diff --git a/packages/agent/src/sync/requester/swm-recovery.ts b/packages/agent/src/sync/requester/swm-recovery.ts index 4b38bfe644..25389dc495 100644 --- a/packages/agent/src/sync/requester/swm-recovery.ts +++ b/packages/agent/src/sync/requester/swm-recovery.ts @@ -97,7 +97,10 @@ export interface RecoverContextGraphSwmDeps { roots: readonly { readonly entity: string }[], metaGraphs: readonly string[], ) => Promise; - /** Replace the active head/operation rows for each exact graph asset. */ + /** + * Replace and insert the active head/operation rows for each exact graph + * asset while holding the production per-KA writer lock. + */ readonly replaceMetaForGraphAssets?: ( assets: readonly GraphScopedSwmRecoveryDescriptor[], ) => Promise; @@ -343,8 +346,9 @@ export async function recoverContextGraphSwm( // A crash between the two therefore retries idempotently; it can never // advertise a head whose graph was only partially transferred. await deps.store.replaceGraph(asset.assertionGraph, [...asset.quads]); - await deps.replaceMetaForGraphAssets?.([descriptor]); - if (verifiedAssetMeta.length > 0) { + if (deps.replaceMetaForGraphAssets) { + await deps.replaceMetaForGraphAssets([descriptor]); + } else if (verifiedAssetMeta.length > 0) { await deps.store.insert([...verifiedAssetMeta]); } incrementallyReadyGraphs.add(graphKey); @@ -487,8 +491,14 @@ export async function recoverContextGraphSwm( if (graphScopedDescriptors.length > 0) { await deps.replaceMetaForGraphAssets?.(graphScopedDescriptors); } - if (processed.verifiedMeta.length > 0) { - await deps.store.insert([...processed.verifiedMeta]); + const replacedGraphScopedMetaKeys = deps.replaceMetaForGraphAssets + ? new Set(graphScopedDescriptors.flatMap((descriptor) => descriptor.metadataQuads).map(quadKey)) + : new Set(); + const remainingVerifiedMeta = processed.verifiedMeta.filter( + (quad) => !replacedGraphScopedMetaKeys.has(quadKey(quad)), + ); + if (remainingVerifiedMeta.length > 0) { + await deps.store.insert([...remainingVerifiedMeta]); } // R2 — hydrate the Rule-4 ownership cache for the recovered roots (parity with diff --git a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts index 08874827fd..2b9d716987 100644 --- a/packages/agent/src/sync/requester/swm-snapshot-materializer.ts +++ b/packages/agent/src/sync/requester/swm-snapshot-materializer.ts @@ -19,8 +19,16 @@ import { withKeyedLocks, workspacePublicQuadsDigest, } from '@origintrail-official/dkg-publisher'; -import type { Quad, TripleStore } from '@origintrail-official/dkg-storage'; +import { + type Quad, + type TripleStore, +} from '@origintrail-official/dkg-storage'; import type { GraphScopedSwmRecoveryDescriptor } from '../graph-scoped-swm-recovery.js'; +import { + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, +} from '../../dkg-agent-constants.js'; +import { buildFinalizedSwmCleanupTaskQuads } from '../../finalized-swm-cleanup-marker.js'; const DKG = 'http://dkg.io/ontology/'; @@ -84,6 +92,20 @@ export interface SharedMemorySnapshotMaterializer { * of equal size apart and would skip a verified newer snapshot. */ isGraphAssetMaterialized(descriptor: GraphScopedSwmRecoveryDescriptor): Promise; + /** + * Re-arm a constant-size GC task when this exact operation carries a local + * finalization tombstone. This is marker bookkeeping only: no graph scan, + * VM verification, deletion, or GC wait occurs on the ingest path. + * + * Call this ONLY on the decision paths that do not go on to + * {@link replaceHeadMetadata} — that replacement re-arms the task from the + * tombstone read it already performs, so calling both duplicates a + * lock-held store read per KA for no added guarantee. + */ + ensureFinalizedCleanupTask( + contextGraphId: string, + descriptor: GraphScopedSwmRecoveryDescriptor, + ): Promise; /** * Atomic whole-graph replace. Replace, not insert: a KA graph is * all-or-nothing and digest-verified; union-insert risks partial or @@ -91,13 +113,18 @@ export interface SharedMemorySnapshotMaterializer { */ replaceGraph(graphUri: string, quads: Quad[]): Promise; /** - * Delete the KA's head rows and every share-operation subject its head - * references (including the descriptor's own, which the caller re-inserts - * from fresh verified metadata). This is the catch-up lane's counterpart of - * gossip's delete-then-insert (`storeKnowledgeAssetWorkspaceHead`) and the - * private recovery lane's `replaceMetaForGraphAssets`: without it the - * append-style meta insert stacks old and new head rows on one subject and - * the durable current head becomes ambiguous. + * Replace the KA's head rows and every share-operation subject its head + * references from fresh verified metadata. This is the catch-up lane's + * counterpart of gossip's delete-then-insert + * (`storeKnowledgeAssetWorkspaceHead`) and the private recovery lane's + * `replaceMetaForGraphAssets`: without it the append-style meta insert stacks + * old and new head rows on one subject and the durable current head becomes + * ambiguous. + * + * Also re-arms the finalized-SWM GC task for this operation: this call is + * what can resurrect a finalized copy, and the tombstone read it performs + * anyway answers that too. Callers therefore need no separate + * {@link ensureFinalizedCleanupTask} on any path that reaches this. */ replaceHeadMetadata( contextGraphId: string, @@ -105,6 +132,192 @@ export interface SharedMemorySnapshotMaterializer { ): Promise; } +/** + * Replace one graph-scoped SWM lifecycle without losing its immutable local + * finalization tombstone — and re-arm the GC task from the same read. + * + * Finalization markers are deliberately local-only and responders filter them + * from synchronized metadata. A blind head/operation replacement therefore + * erased the metadata-only lifecycle record needed to re-arm background GC. + * Preserve the operation tombstone only for the exact incoming operation + * subject. The independent GC task is stored on its own subject and is not + * touched by this replacement. + * + * Re-arming HERE rather than ahead of the caller's decisions is what keeps the + * ingest path at one tombstone read: this replacement is itself the act that + * can resurrect a finalized SWM copy, and the read it already performs answers + * both questions. Reading at the last possible moment before the delete also + * narrows — it cannot close — the window in which a concurrently written + * tombstone is destroyed, because `FinalizationHandler` writes markers without + * taking the per-KA SWM writer lock. + * + * The verified replacement metadata, any retained operation tombstone, and the + * constant-size GC task are inserted in the same store call after the old + * lifecycle is removed. The task's own subject is never in the delete set, so + * the insert is an idempotent re-arm. Callers MUST hold the canonical per-KA + * SWM writer lock across this entire operation. + */ +export async function replaceGraphScopedSwmHeadMetadata(params: { + store: TripleStore; + contextGraphId: string; + descriptor: GraphScopedSwmRecoveryDescriptor; + sourcePrefix: string; + insertReplacementMetadata: (quads: readonly Quad[]) => Promise; +}): Promise { + const { + store, + contextGraphId, + descriptor, + sourcePrefix, + insertReplacementMetadata, + } = params; + const preserved = await readExactFinalizedOperationTombstone({ + store, + contextGraphId, + descriptor, + sourcePrefix, + }); + + // Collect every share operation the head currently references — via the + // BOUND head subject, then per-candidate bound-subject ASKs. The kaUal guard + // prevents a corrupt cross-KA reference from deleting another KA's metadata. + const shareIds = await store.query( + `SELECT DISTINCT ?op WHERE { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(descriptor.headSubject)}> <${DKG}shareOperationId> ?op } }`, + { priority: 'background', source: `${sourcePrefix}.findOperations` }, + ); + const operationSubjects = new Set([descriptor.operationSubject]); + if (shareIds.type === 'bindings') { + for (const row of shareIds.bindings) { + const shareId = literalValue(row?.['op']); + if (!shareId) continue; + const candidate = `urn:dkg:share:${contextGraphId}:${shareId}`; + if (operationSubjects.has(candidate)) continue; + const ownedByThisKa = await store.query( + `ASK { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(candidate)}> <${DKG}kaUal> <${assertSafeIri(descriptor.kaUal)}> } }`, + { priority: 'background', source: `${sourcePrefix}.checkOperation` }, + ); + if (ownedByThisKa.type === 'boolean' && ownedByThisKa.value) { + operationSubjects.add(candidate); + } + } + } + + await store.deleteByPattern( + { graph: descriptor.metaGraph, subject: descriptor.headSubject }, + { priority: 'background', source: `${sourcePrefix}.deleteHead` }, + ); + for (const operationSubject of operationSubjects) { + await store.deleteByPattern( + { graph: descriptor.metaGraph, subject: operationSubject }, + { priority: 'background', source: `${sourcePrefix}.deleteOperation` }, + ); + } + const replacementQuads = [ + ...descriptor.metadataQuads, + ...preserved.tombstone, + ...preserved.cleanupTask, + ]; + if (replacementQuads.length > 0) { + await insertReplacementMetadata(replacementQuads); + } +} + +/** + * Cheap existence gate for {@link readExactFinalizedOperationTombstone}. + * + * This is the same bound (graph, subject, predicate) pattern the CONSTRUCT + * below matches on, reduced to an ASK: no `?root` binding means that CONSTRUCT + * has no solutions, returns no quads, and the reader already gives up on + * `roots.length !== 1`. So a false here is exactly the reader's empty result — + * a strictly weaker query over the identical pattern, not a policy gate, and + * therefore incapable of missing a finalized marker. It is a live store read, + * so nothing about it can go stale, and nothing the GC reads depends on it. + */ +async function hasFinalizedOperationTombstone(params: { + store: TripleStore; + descriptor: GraphScopedSwmRecoveryDescriptor; + sourcePrefix: string; +}): Promise { + const { store, descriptor, sourcePrefix } = params; + const result = await store.query( + `ASK { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + { + priority: 'background', + source: `${sourcePrefix}.askFinalizedOperationTombstone`, + }, + ); + return result.type === 'boolean' && result.value; +} + +async function readExactFinalizedOperationTombstone(params: { + store: TripleStore; + contextGraphId: string; + descriptor: GraphScopedSwmRecoveryDescriptor; + sourcePrefix: string; +}): Promise<{ tombstone: Quad[]; cleanupTask: Quad[] }> { + const { store, descriptor, sourcePrefix } = params; + const markerResult = await store.query( + `CONSTRUCT { <${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . ` + + `<${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } WHERE { ` + + `GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` + + `<${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root . ` + + `OPTIONAL { <${assertSafeIri(descriptor.operationSubject)}> ` + + `<${FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE}> ?markedAt } } }`, + { + priority: 'background', + source: `${sourcePrefix}.readFinalizedOperationTombstone`, + }, + ); + if (markerResult.type !== 'quads') return { tombstone: [], cleanupTask: [] }; + const markers = markerResult.quads.map((quad) => ({ + ...quad, + graph: descriptor.metaGraph, + })); + const roots = markers.filter((quad) => + quad.subject === descriptor.operationSubject + && quad.predicate === FINALIZED_SWM_CLEANUP_ROOT_PREDICATE); + const markedAtRows = markers.filter((quad) => + quad.subject === descriptor.operationSubject + && quad.predicate === FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE); + if (roots.length !== 1) { + return { tombstone: [], cleanupTask: [] }; + } + const rootHex = literalValue(roots[0]!.object); + if (!rootHex || !/^0x[0-9a-fA-F]{64}$/.test(rootHex)) { + return { tombstone: [], cleanupTask: [] }; + } + // Repeated finalization evidence can append several timestamp values to the + // immutable operation subject. Keep the earliest valid value for backlog + // age and normalize the replacement back to one row. + const earliestMarkedAt = markedAtRows + .map((quad) => ({ quad, value: literalValue(quad.object) })) + .filter((entry): entry is { quad: Quad; value: string } => + entry.value !== undefined && Number.isFinite(Date.parse(entry.value))) + .sort((a, b) => Date.parse(a.value) - Date.parse(b.value))[0]; + const markedAtIso = earliestMarkedAt?.value ?? new Date().toISOString(); + return { + tombstone: [roots[0]!, ...(earliestMarkedAt ? [earliestMarkedAt.quad] : [])], + cleanupTask: buildFinalizedSwmCleanupTaskQuads({ + contextGraphId: params.contextGraphId, + subGraphName: descriptor.subGraphName, + head: { + ...descriptor, + publicTripleCount: descriptor.publicQuadsCount, + }, + expectedMerkleRootHex: rootHex, + metaGraph: descriptor.metaGraph, + markedAtIso, + }), + }; +} + /** * Build the production materializer over the agent's own store, lock map and * list-cache invalidation hook. @@ -117,6 +330,12 @@ export function createSharedMemorySnapshotMaterializer(deps: { */ writeLocks: Map>; invalidateListContextGraphsCache: () => void; + /** + * The same filtered insert seam used by aggregate SWM pages. Replacement + * metadata is peer-controlled too, so bypassing this guard can re-arm the + * oversized-literal retry loop after the old head has been removed. + */ + insertReplacementMetadata: (quads: readonly Quad[]) => Promise; }): SharedMemorySnapshotMaterializer { return { withKaWriteLock: (contextGraphId, subGraphName, kaUal, fn) => @@ -178,6 +397,26 @@ export function createSharedMemorySnapshotMaterializer(deps: { return workspacePublicQuadsDigest(stored) === descriptor.publicQuadsDigest; }, + ensureFinalizedCleanupTask: async (contextGraphId, descriptor) => { + // Most descriptors were never finalized, so answer that with one bound + // ASK instead of materializing a CONSTRUCT result per KA. + if (!await hasFinalizedOperationTombstone({ + store: deps.store, + descriptor, + sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + })) return; + const preserved = await readExactFinalizedOperationTombstone({ + store: deps.store, + contextGraphId, + descriptor, + sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + }); + if (preserved.cleanupTask.length > 0) { + await deps.insertReplacementMetadata(preserved.cleanupTask); + deps.invalidateListContextGraphsCache(); + } + }, + replaceGraph: async (graphUri, quads) => { // Deliberately NOT routed through the sync lane's guarded union insert: // a KA graph is all-or-nothing and digest-verified, so it must land via @@ -193,43 +432,14 @@ export function createSharedMemorySnapshotMaterializer(deps: { }, replaceHeadMetadata: async (contextGraphId, descriptor) => { - // Collect every share operation the head currently references — via the - // BOUND head subject, then per-candidate bound-subject ASKs, so no query - // scans the meta bucket. The kaUal guard mirrors the recovery lane's - // `replaceMetaForGraphAssets` join: a head row pointing at another KA's - // operation must not delete that KA's metadata. - const shareIds = await deps.store.query( - `SELECT DISTINCT ?op WHERE { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` - + `<${assertSafeIri(descriptor.headSubject)}> <${DKG}shareOperationId> ?op } }`, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.findOperations' }, - ); - const operationSubjects = new Set([descriptor.operationSubject]); - if (shareIds.type === 'bindings') { - for (const row of shareIds.bindings) { - const shareId = literalValue(row?.['op']); - if (!shareId) continue; - const candidate = `urn:dkg:share:${contextGraphId}:${shareId}`; - if (operationSubjects.has(candidate)) continue; - const ownedByThisKa = await deps.store.query( - `ASK { GRAPH <${assertSafeIri(descriptor.metaGraph)}> { ` - + `<${assertSafeIri(candidate)}> <${DKG}kaUal> <${assertSafeIri(descriptor.kaUal)}> } }`, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.checkOperation' }, - ); - if (ownedByThisKa.type === 'boolean' && ownedByThisKa.value) { - operationSubjects.add(candidate); - } - } - } - await deps.store.deleteByPattern( - { graph: descriptor.metaGraph, subject: descriptor.headSubject }, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.deleteHead' }, - ); - for (const operationSubject of operationSubjects) { - await deps.store.deleteByPattern( - { graph: descriptor.metaGraph, subject: operationSubject }, - { priority: 'background', source: 'agent.sharedMemorySync.snapshotMaterializer.deleteOperation' }, - ); - } + await replaceGraphScopedSwmHeadMetadata({ + store: deps.store, + contextGraphId, + descriptor, + sourcePrefix: 'agent.sharedMemorySync.snapshotMaterializer', + insertReplacementMetadata: deps.insertReplacementMetadata, + }); + deps.invalidateListContextGraphsCache(); }, }; } diff --git a/packages/agent/src/sync/responder/graph-plan.ts b/packages/agent/src/sync/responder/graph-plan.ts index 9e59a89a26..d7848e12a4 100644 --- a/packages/agent/src/sync/responder/graph-plan.ts +++ b/packages/agent/src/sync/responder/graph-plan.ts @@ -30,7 +30,13 @@ import { type SyncResponderSnapshotBudget, } from './snapshot-budget.js'; import { estimateStringRowHeapBytes } from '../memory-telemetry.js'; -import { SYNC_BYTE_BUDGET_RESPONSE_BYTES } from '../../dkg-agent-constants.js'; +import { + FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, + SYNC_BYTE_BUDGET_RESPONSE_BYTES, +} from '../../dkg-agent-constants.js'; import type { ChangelogSyncResponse, ChangelogDeltaRecord } from '../changelog/wire.js'; import { durableMetaDelegationSubjectAdmissionExpression } from './durable-meta-admission.js'; import { exactAssetFilterKey } from '../exact-assets.js'; @@ -53,6 +59,31 @@ const DKG_CONTENT_SCOPE_VERSION = `${DKG}contentScopeVersion`; const DKG_KA_UAL = `${DKG}kaUal`; const DKG_ASSERTION_VERSION = `${DKG}assertionVersion`; const DKG_SHARE_OPERATION_ID = `${DKG}shareOperationId`; +// DERIVED from the canonical constants the marker WRITER uses, never retyped. +// These four decide what stays local: three predicates the row filter strips, +// and the task type the subject-level FILTER NOT EXISTS matches. A local +// literal that drifted from the writer would keep filtering the old IRI and +// silently advertise local GC bookkeeping to peers — and per the mutation +// matrix on this filter, in-process-only drift survives testing, so nothing +// would catch it. Short aliases only, so the SPARQL templates stay readable. +const DKG_FINALIZED_SWM_CLEANUP_ROOT = FINALIZED_SWM_CLEANUP_ROOT_PREDICATE; +const DKG_FINALIZED_SWM_CLEANUP_MARKED_AT = FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE; +const DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT = FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT_PREDICATE; +const DKG_FINALIZED_SWM_CLEANUP_TASK = FINALIZED_SWM_CLEANUP_TASK_TYPE; +/** In-process mirror of the store-side predicate filter below. */ +const LOCAL_FINALIZED_SWM_CLEANUP_PREDICATES = new Set([ + DKG_FINALIZED_SWM_CLEANUP_ROOT, + DKG_FINALIZED_SWM_CLEANUP_MARKED_AT, + DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT, +]); +const localFinalizedSwmCleanupRowFilter = (subject: string, predicate: string): string => ` + FILTER(${predicate} NOT IN ( + <${DKG_FINALIZED_SWM_CLEANUP_ROOT}>, + <${DKG_FINALIZED_SWM_CLEANUP_MARKED_AT}>, + <${DKG_FINALIZED_SWM_CLEANUP_HEAD_FINGERPRINT}> + )) + FILTER NOT EXISTS { ${subject} <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_FINALIZED_SWM_CLEANUP_TASK}> } +`; const DKG_ASSERTION_GRAPH = `${DKG}assertionGraph`; const DKG_ASSERTION_NAME = `${DKG}assertionName`; const DKG_MEMORY_LAYER = `${DKG}memoryLayer`; @@ -2600,9 +2631,18 @@ async function readBoundedSwmMetaSnapshot( } let result; try { + // Filter local finalized-cleanup rows AT THE STORE, like the fresh/TTL + // lanes already do. Stripping them in-process afterwards is not enough: + // every such row would first be accumulated against the peer-serving + // snapshot budget, so purely local GC bookkeeping could push a large + // context graph over SYNC_RESPONDER_SNAPSHOT_BUILD_MAX_ROWS/_BYTES and + // make it unsyncable (#1847/#1868 snapshotBudgetError). result = await store.query(` SELECT ?s ?p ?o WHERE { - GRAPH <${assertSafeIri(graph)}> { ?s ?p ?o } + GRAPH <${assertSafeIri(graph)}> { + ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} + } } LIMIT ${remainingRows + 1} `, { @@ -2659,10 +2699,30 @@ function filterSwmMetaSnapshotRows( rows: readonly SyncRow[], cutoffIso: string | null, ): SyncRow[] { - if (cutoffIso == null) return [...rows].sort(compareRows); - const cutoffMs = Date.parse(cutoffIso); - if (!Number.isFinite(cutoffMs)) return []; + // An unparseable cutoff admits nothing, so settle it BEFORE any O(rows) + // work — otherwise the responder indexes and scans the whole snapshot only + // to throw it away. + const cutoffMs = cutoffIso == null ? Number.NaN : Date.parse(cutoffIso); + if (cutoffIso != null && !Number.isFinite(cutoffMs)) return []; + + // Finalized-SWM markers/tasks are local maintenance state. Strip only those + // rows; the active SWM lifecycle and its payload remain syncable until the + // independent idle GC safely removes them. This must stay ABOVE the + // TTL-disabled return: that lane is served from this function too, and an + // unfiltered return there advertises local GC metadata to peers. + const cleanupTaskSubjects = new Set(); + for (const row of rows) { + if (row.p === DKG_ONTOLOGY.RDF_TYPE && row.o === DKG_FINALIZED_SWM_CLEANUP_TASK) { + cleanupTaskSubjects.add(row.s); + } + } + const syncableRows = rows.filter((row) => + !cleanupTaskSubjects.has(row.s) && !LOCAL_FINALIZED_SWM_CLEANUP_PREDICATES.has(row.p)); + if (cutoffIso == null) return syncableRows.sort(compareRows); + // TTL lane only: the per-subject index exists solely to answer the freshness + // and tuple-key questions below, so it is built after the lanes that cannot + // use it have already returned. const bySubject = new Map(); for (const row of rows) { const bucket = bySubject.get(row.s) ?? []; @@ -2694,7 +2754,6 @@ function filterSwmMetaSnapshotRows( } return keys; }; - const admitted = new Set(); const freshOperationKeys = new Set(); for (const [subject] of bySubject) { @@ -2710,12 +2769,14 @@ function filterSwmMetaSnapshotRows( if (tupleKeys(subject).some((key) => freshOperationKeys.has(key))) admitted.add(subject); } - return rows.filter((row) => admitted.has(row.s)).sort(compareRows); + return syncableRows.filter((row) => admitted.has(row.s)).sort(compareRows); } /** - * Legacy UNFILTERED store-paged compatibility path (cutoffIso == null sessions - * only). The former TTL variant of this query — DISTINCT + a six-predicate + * Legacy TTL-unfiltered store-paged compatibility path (cutoffIso == null + * sessions only). Local finalized-cleanup metadata is filtered while the SWM + * lifecycle itself remains syncable until idle GC removes it. The former TTL + * variant of this query — DISTINCT + a six-predicate * UNION join + global `ORDER BY ?g ?s ?p ?o` re-evaluated with a growing * OFFSET per page over a mutable graph family — was the #1847 store-melter and * is deliberately DELETED, not gated: TTL-filtered sessions page from the @@ -2734,12 +2795,13 @@ async function readSwmMetaRowsPage( const swmMetaValues = graphValues(swmMetaGraphs); if (!swmMetaValues) return []; // sparql-scan-allow: R2 -- ?g is bound by a finite VALUES list of pre-admitted SWM meta graph IRIs - // sparql-scan-allow: R3 -- pre-existing legacy (cutoff-less) compatibility lane, unchanged behavior; TTL sessions page from the session plan instead (#1847) + // sparql-scan-allow: R3 -- legacy cutoff-less compatibility lane; TTL sessions page from the session plan instead (#1847) const res = await store.query(` SELECT DISTINCT ?g ?s ?p ?o WHERE { VALUES ?g { ${swmMetaValues} } GRAPH ?g { ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} } } ORDER BY ?g ?s ?p ?o @@ -2838,6 +2900,7 @@ async function readFreshSwmMetaSubjects( SELECT DISTINCT ?s WHERE { GRAPH <${assertSafeIri(graph)}> { ?s <${DKG_PUBLISHED_AT}> ?ts . + FILTER NOT EXISTS { ?s <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_FINALIZED_SWM_CLEANUP_TASK}> } ${cutoffFilter} } } @@ -2850,6 +2913,7 @@ async function readFreshSwmMetaSubjects( <${DKG_KA_UAL}> ?headUal ; <${DKG_ASSERTION_VERSION}> ?headVersion ; <${DKG_SHARE_OPERATION_ID}> ?shareId . + FILTER NOT EXISTS { ?s <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_FINALIZED_SWM_CLEANUP_TASK}> } ?headOperation <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_WORKSPACE_OPERATION}> ; <${DKG_CONTENT_SCOPE_VERSION}> ${GRAPH_KA_CONTENT_SCOPE_VERSION} ; <${DKG_KA_UAL}> ?headUal ; @@ -2882,7 +2946,10 @@ async function countFreshSwmMetaSubjectRows( res = await store.query(` SELECT ?s (COUNT(*) AS ?count) WHERE { VALUES ?s { ${subjectValues(chunk)} } - GRAPH <${assertSafeIri(graph)}> { ?s ?p ?o } + GRAPH <${assertSafeIri(graph)}> { + ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} + } } GROUP BY ?s `, { @@ -3029,7 +3096,10 @@ async function readFreshSwmMetaSubjectWindowRows( const res = await store.query(` SELECT ?s ?p ?o WHERE { VALUES ?s { ${subjectValues(chunk.map((entry) => entry.subject))} } - GRAPH <${assertSafeIri(graph)}> { ?s ?p ?o } + GRAPH <${assertSafeIri(graph)}> { + ?s ?p ?o . + ${localFinalizedSwmCleanupRowFilter('?s', '?p')} + } } `, { ...syncResponderStoreOptions(signal, 'sync.responder.readFreshSwmMetaSubjectRows'), diff --git a/packages/agent/test/agent.part-16.test.ts b/packages/agent/test/agent.part-16.test.ts index 2587ee8993..e46a2e1990 100644 --- a/packages/agent/test/agent.part-16.test.ts +++ b/packages/agent/test/agent.part-16.test.ts @@ -125,40 +125,51 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => // `insertedTriples` count exposed by `syncFromPeer` / `syncSharedMemoryFromPeer`. // Replace those with recorders so we can assert both the call shape and // the reported totals without spinning up a remote peer. - const syncFromPeerDetailed = recorder(async () => ({ - insertedTriples: 5, - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 5, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 0, - metaOnlyResponses: 0, - dataRejectedMissingMeta: 0, - rejectedKcs: 0, - failedPeers: 0, - })); + const lifecycleOrder: string[] = []; + const syncFromPeerDetailed = recorder(async () => { + lifecycleOrder.push('durable'); + return { + insertedTriples: 5, + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 5, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 0, + metaOnlyResponses: 0, + dataRejectedMissingMeta: 0, + rejectedKcs: 0, + failedPeers: 0, + }; + }); (agent as any).syncFromPeerDetailed = syncFromPeerDetailed; - const syncSharedMemoryFromPeerDetailed = recorder(async () => ({ - insertedTriples: 2, - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 2, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 0, - droppedDataTriples: 0, - failedPeers: 0, - })); + const syncSharedMemoryFromPeerDetailed = recorder(async () => { + lifecycleOrder.push('shared-memory'); + return { + insertedTriples: 2, + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 2, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 0, + droppedDataTriples: 0, + failedPeers: 0, + }; + }); (agent as any).syncSharedMemoryFromPeerDetailed = syncSharedMemoryFromPeerDetailed; + const wakeFinalizedSwmCleanup = recorder(() => { + lifecycleOrder.push('cleanup-wake'); + }); + (agent as any).wakeFinalizedSwmCleanup = wakeFinalizedSwmCleanup; const result = await agent.syncContextGraphFromConnectedPeers('runtime-contextGraph', { includeSharedMemory: true, @@ -186,6 +197,8 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => expect(result.diagnostics.noProtocolPeers).toBe(0); expect(result.diagnostics.durable.failedPeers).toBe(0); expect(result.diagnostics.sharedMemory.failedPeers).toBe(0); + expect(wakeFinalizedSwmCleanup.calls).toEqual([[]]); + expect(lifecycleOrder).toEqual(['durable', 'shared-memory', 'cleanup-wake']); expect(agent.getSubscribedContextGraphs().get('runtime-contextGraph')).toMatchObject({ synced: true, sharedMemorySynced: true, diff --git a/packages/agent/test/finalized-swm-cleanup-rotation.test.ts b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts new file mode 100644 index 0000000000..3550053d07 --- /dev/null +++ b/packages/agent/test/finalized-swm-cleanup-rotation.test.ts @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Rotation-cursor behaviour of the idle finalized-SWM GC. + * + * A sweep that yields mid-rotation persists its unvisited tail, so the next + * slice continues instead of re-walking the same prefix forever. That makes + * `backlogDepth` the delicate part: with a rotating start, a naive per-sweep + * sum would be a partial number whose meaning changes every sweep. The service + * instead accumulates per-context-graph contributions and publishes only when + * the rotation closes, so the metric can be STALE but never PARTIAL. + * + * The assertions here are therefore about the published totals and about which + * context graphs each slice actually entered. Asserting only that the cursor + * advanced is much weaker: it passes with the accumulator committed in the + * wrong place, which is the mutation that silently inflates the SLO surface. + * + * VERIFYING THIS SUITE STILL BITES. Neutralise the cursor at its DECLARATION, + * not at its assignment sites: + * + * private rotationPendingDiscarded: string[] | null = null; + * private get rotationPending(): string[] | null { return null; } + * private set rotationPending(_v: string[] | null) {} + * + * The sweep loop has repeatedly grown new persist sites — a fault path, then a + * task cursor — and a mutant that enumerates them silently weakens each time + * one is added: the enumeration was complete when written, the code grew + * underneath it, and nothing fails to announce that. Mutating the single + * declaration cannot be outgrown by new call sites, so it stays valid across + * restructures of the loop. + * + * This is NOT the same as a blanket mutant, and the difference decides whether + * a result means anything: + * + * - A BLANKET mutant stands in for many PROPERTIES. That is the failure to + * avoid — four conversion sites killed by one test, three predicates + * asserted in one case. Its kill tells you something broke, not what. + * - A SITE-PROOF mutant covers ONE property and is merely robust to that + * property gaining implementation sites. + * + * The check is whether the kill set stays narrow and specific: the mutation + * above kills the resume test and nothing else. If a site-proof mutant starts + * reddening half the suite it has become blanket, and that is the signal to + * split it rather than to celebrate the kill. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { + GraphManager, + OxigraphStore, + type Quad, + type QueryOptions, +} from '@origintrail-official/dkg-storage'; +import { + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from '../src/dkg-agent-constants.js'; +import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; + +/** + * `runSweep` derives TWO deadlines from `wallClockBudgetMs`: one from the + * injected `now`, and one from `AbortSignal.timeout(...)` on REAL wall time. + * The second is threaded into the store and aborts its queries, and nothing in + * a test can control it. + * + * The budget must therefore exceed any plausible real slice duration. With a + * small budget, a slow machine, a loaded CI box or a neighbouring worktree + * makes the real timer fire first and the sweep yields at a boundary the + * injected clock never chose — reddening these tests for reasons unrelated to + * the code they pin. Measured directly: with the injected clock frozen and a + * 100 ms budget, a 150 ms store query still returns `budgetExhausted: true`. + * + * The injected clock alone must decide every deadline yield here, so the tick + * is expressed as a fraction of the budget: two ticks overrun one slice, one + * does not. Do not shrink these to make the arithmetic look tidier. + */ +const SLICE_BUDGET_MS = 60_000; +const SLICE_TICK_MS = Math.ceil(SLICE_BUDGET_MS * 0.6); + +const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; +const XSD_DATE_TIME = 'http://www.w3.org/2001/XMLSchema#dateTime'; +const MARKER_ROOT = JSON.stringify(`0x${'11'.repeat(32)}`); + +/** + * Backlog markers only. The discovery query additionally requires kaUal, + * assertionVersion, shareOperationId, assertionGraph and headFingerprint, so + * these rows are counted by `inspectBacklog` and ignored by cleanup — which is + * what keeps these tests about rotation rather than about deletion. + */ +function markerQuads(label: string, metaGraph: string, count: number): Quad[] { + const quads: Quad[] = []; + for (let index = 0; index < count; index += 1) { + const subject = `urn:dkg:finalized-swm-cleanup:${label}-${index}`; + quads.push( + { subject, predicate: RDF_TYPE, object: FINALIZED_SWM_CLEANUP_TASK_TYPE, graph: metaGraph }, + { subject, predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, object: MARKER_ROOT, graph: metaGraph }, + { + subject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: `"${new Date(1_700_000_000_000 + index * 1_000).toISOString()}"^^<${XSD_DATE_TIME}>`, + graph: metaGraph, + }, + ); + } + return quads; +} + +function installPressureSwitch(store: OxigraphStore): { busy: boolean } { + const state = { busy: false }; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: state.busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + return state; +} + +describe('finalized SWM cleanup rotation cursor', () => { + /** + * The property the whole accumulate-across-rotation design exists to hold: a + * slice that yields part-way through a context graph must discard that + * context graph's partial contribution, so the re-measure on resume cannot + * double-count it. + * + * 20 markers across two meta graphs of one context graph. The first slice + * measures one meta graph and then yields, the second measures both. The + * published total must be 20 — committing each meta graph straight into the + * rotation accumulator instead of at the context-graph boundary yields 30. + */ + it('discards a partial context-graph measurement rather than double-counting it on resume', async () => { + const store = new OxigraphStore(); + const pressure = installPressureSwitch(store); + const graphManager = new GraphManager(store); + const rootMeta = graphManager.sharedMemoryMetaUri('cg-partial'); + const subMeta = graphManager.sharedMemoryMetaUri('cg-partial', 'named'); + await store.insert(markerQuads('root', rootMeta, 10)); + await store.insert(markerQuads('sub', subMeta, 10)); + + // Pressure appears once the first meta graph has been measured, so the + // slice yields at the top of the second meta-graph iteration. + let backlogReads = 0; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + const result = await originalQuery(query, options); + if (options?.source === 'agent.finalizedSwmCleanup.backlog') { + backlogReads += 1; + if (backlogReads === 1) pressure.busy = true; + } + return result; + }); + + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => ['cg-partial'], + listSharedMemoryMetaGraphs: async () => [rootMeta, subMeta], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + stale: true, + // Nothing has ever been published, and the 10 markers measured in this + // slice are NOT it — that would be a partial sum. + backlogDepth: 0, + }); + expect(backlogReads).toBe(1); + + pressure.busy = false; + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + stale: false, + backlogDepth: 20, + }); + // Three reads total: one discarded, then both meta graphs re-measured. + expect(backlogReads).toBe(3); + querySpy.mockRestore(); + }); + + /** + * A yielding sweep must resume at its unvisited tail. Before the cursor, + * every retry re-walked the same prefix, so markers in later context graphs + * were never discovered no matter how many times the worker woke. + * + * Four context graphs, 10 markers each, enumeration costing one tick against + * a slice that fits fewer than two. Context graphs legitimately repeat across + * slices — a graph entered but not finished is re-measured whole — so the + * assertion is on the entry sequence and on the single published total, not + * on disjoint sets. + */ + it('resumes at the unvisited tail and publishes a whole-node total only when the rotation closes', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const contextGraphIds = ['cg-a', 'cg-b', 'cg-c', 'cg-d']; + const metaByContextGraph = new Map(); + for (const contextGraphId of contextGraphIds) { + const meta = graphManager.sharedMemoryMetaUri(contextGraphId); + metaByContextGraph.set(contextGraphId, meta); + await store.insert(markerQuads(contextGraphId, meta, 10)); + } + + const clock = { now: 1_000_000 }; + const entered: string[][] = []; + let currentSweep: string[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: SLICE_BUDGET_MS, + listContextGraphIds: async () => [...contextGraphIds], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + currentSweep.push(contextGraphId); + clock.now += SLICE_TICK_MS; + return [metaByContextGraph.get(contextGraphId)!]; + }, + }); + + const runSlice = async () => { + currentSweep = []; + const result = await service.runSweep(); + entered.push(currentSweep); + return result; + }; + + await expect(runSlice()).resolves.toMatchObject({ budgetExhausted: true, stale: true, backlogDepth: 0 }); + await expect(runSlice()).resolves.toMatchObject({ budgetExhausted: true, stale: true, backlogDepth: 0 }); + await expect(runSlice()).resolves.toMatchObject({ budgetExhausted: true, stale: true, backlogDepth: 0 }); + // The rotation closes here: every context graph has been measured exactly + // once, so 4 x 10 markers is a whole-node total. + const closing = await runSlice(); + expect(closing).toMatchObject({ pressureSkipped: false, stale: false, backlogDepth: 40 }); + expect(closing.budgetExhausted).toBeUndefined(); + + // Seven context-graph entries produced a total of 40, not 70: the three + // graphs entered twice contributed once each. + expect(entered).toEqual([ + ['cg-a', 'cg-b'], + ['cg-b', 'cg-c'], + ['cg-c', 'cg-d'], + ['cg-d'], + ]); + expect(entered.flat()).toHaveLength(7); + + // A fresh rotation opens at the head again, republishing the last complete + // total as stale rather than resetting it to zero. + await expect(runSlice()).resolves.toMatchObject({ stale: true, backlogDepth: 40 }); + expect(entered[4]).toEqual(['cg-a', 'cg-b']); + }); + + /** + * Forward progress. A context graph whose enumeration cannot finish inside + * one slice would otherwise hold the cursor forever and starve every context + * graph behind it — the same "later graphs are never reached" failure the + * cursor exists to fix. + */ + it('rotates a head that repeatedly cannot be finished so the tail is still served', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const slowMeta = graphManager.sharedMemoryMetaUri('cg-slow'); + const metaB = graphManager.sharedMemoryMetaUri('cg-b'); + const metaC = graphManager.sharedMemoryMetaUri('cg-c'); + await store.insert(markerQuads('slow', slowMeta, 5)); + await store.insert(markerQuads('b', metaB, 5)); + await store.insert(markerQuads('c', metaC, 5)); + + const clock = { now: 1_000_000 }; + const entered: string[][] = []; + let currentSweep: string[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: SLICE_BUDGET_MS, + listContextGraphIds: async () => ['cg-slow', 'cg-b', 'cg-c'], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + currentSweep.push(contextGraphId); + // The head alone costs more than a whole slice, so it can never finish. + clock.now += contextGraphId === 'cg-slow' ? SLICE_BUDGET_MS * 2 : Math.ceil(SLICE_BUDGET_MS * 0.1); + if (contextGraphId === 'cg-slow') return [slowMeta]; + return [contextGraphId === 'cg-b' ? metaB : metaC]; + }, + }); + + for (let slice = 0; slice < 3; slice += 1) { + currentSweep = []; + await service.runSweep(); + entered.push(currentSweep); + } + + expect(entered[0]).toEqual(['cg-slow']); + expect(entered[1]).toEqual(['cg-slow']); + // The head steps aside after TWO failed slices and keeps its place in the + // rotation, so the graphs behind it are finally reached — and it is still + // retried in the same pass rather than dropped. + expect(entered[2]).toEqual(['cg-b', 'cg-c', 'cg-slow']); + }); + + /** + * Owner/name context graphs (`/`) must stay reachable by GC + * discovery. Nothing about that is obvious from the code, and it has already + * survived two independent near-misses: an enumeration source that drops ids + * containing `/`, and a listing mode that drops context graphs with no + * explicit privacy policy. Either would have made these graphs invisible to + * the GC and left their finalized SWM copies forever — #1996 reintroduced by + * the back door, in the subsystem meant to close it. + * + * This pins the service side: an id containing `/` must survive graph-URI + * derivation, discovery and backlog measurement. Which enumeration source + * feeds `listContextGraphIds` is production wiring and is covered separately + * by the agent-level TTL suite. + */ + it('discovers and measures an owner/name context graph', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const ownerNameId = `${'0x1111111111111111111111111111111111111111'}/public-finalized-cleanup`; + const rootMeta = graphManager.sharedMemoryMetaUri(ownerNameId); + const subMeta = graphManager.sharedMemoryMetaUri(ownerNameId, 'named-slice'); + expect(rootMeta).toContain(ownerNameId); + await store.insert(markerQuads('owner-root', rootMeta, 4)); + await store.insert(markerQuads('owner-sub', subMeta, 2)); + + const enumerated: string[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [ownerNameId], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + enumerated.push(contextGraphId); + return [rootMeta, subMeta]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + stale: false, + // Both the root and the named sub-graph meta graph of the owner/name CG + // were measured; a `/`-rejecting path anywhere here reports 0. + backlogDepth: 6, + }); + expect(enumerated).toEqual([ownerNameId]); + }); + + /** + * A context graph deleted mid-rotation must not strand the rotation. The + * pending tail is re-filtered against a fresh enumeration each slice. + */ + it('drops a context graph that disappears mid-rotation without stranding the rotation', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const metaA = graphManager.sharedMemoryMetaUri('cg-a'); + const metaB = graphManager.sharedMemoryMetaUri('cg-b'); + await store.insert(markerQuads('a', metaA, 7)); + await store.insert(markerQuads('b', metaB, 3)); + + const clock = { now: 1_000_000 }; + let live = ['cg-a', 'cg-b']; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: SLICE_BUDGET_MS, + listContextGraphIds: async () => [...live], + listSharedMemoryMetaGraphs: async (contextGraphId) => { + clock.now += SLICE_TICK_MS; + return [contextGraphId === 'cg-a' ? metaA : metaB]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ stale: true, backlogDepth: 0 }); + + // cg-b is deleted while it is still the pending tail. + live = ['cg-a']; + await expect(service.runSweep()).resolves.toMatchObject({ + stale: false, + // Only the graphs that still exist contribute; the rotation still closes. + backlogDepth: 7, + }); + }); + + /** + * The per-context-graph enumeration is the call that scales with graph count, + * so it must reach the store on the background lane and carry the slice's + * deadline. An absent priority defaults to `normal`, which turned the GC's own + * discovery into a foreground scan on a cold graph-set index. + */ + it('runs the per-context-graph meta enumeration on the background lane with the slice deadline', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const seen: QueryOptions[] = []; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => ['cg-lane'], + listSharedMemoryMetaGraphs: async (_contextGraphId, options) => { + seen.push(options); + return []; + }, + }); + + await service.runSweep(); + + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ + priority: 'background', + source: 'agent.finalizedSwmCleanup.discover', + }); + expect(seen[0]!.signal).toBeInstanceOf(AbortSignal); + }); +}); diff --git a/packages/agent/test/finalized-swm-cleanup-sweep.test.ts b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts new file mode 100644 index 0000000000..cc7dba1758 --- /dev/null +++ b/packages/agent/test/finalized-swm-cleanup-sweep.test.ts @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Control-flow guards of the idle finalized-SWM GC slice. + * + * `FinalizedSwmCleanupService.runSweep()` is the only discovery boundary the + * garbage collector owns, and the reviewer's acceptance criteria are about what + * it *refuses* to do: under sustained foreground load it must perform no + * discovery and no payload scan at all, and once its wall-clock budget is spent + * it must yield the slice rather than keep enumerating. + * + * Those are properties of the in-loop `return`s, not of the happy path, so the + * assertions here are deliberately about work NOT attempted — every test pins a + * specific gate by observing the next store read or discovery closure that would + * run if the gate were removed. A test that only asserted the returned + * `FinalizedSwmCleanupSweepResult` would pass with the gates deleted, because + * the callee-side guards inside `cleanupMetaGraph`/`inspectBacklog` produce the + * same result shape one step later. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { + GraphManager, + OxigraphStore, + StoreSchedulerBusyError, + type TripleStore, +} from '@origintrail-official/dkg-storage'; +import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; +import { FinalizedSwmCleanupWorker } from '../src/finalized-swm-cleanup-worker.js'; + +const CG = 'sweep-gate-cg'; + +/** Toggleable foreground pressure; the service only reads the snapshot. */ +function installPressureSwitch(store: TripleStore): { busy: boolean } { + const state = { busy: false }; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: state.busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + return state; +} + +/** + * Records every store *method call* (not property read) so a test can assert + * that a pass touched the store zero times. `asGraphWriteGenSource` only probes + * `typeof store.getWriteGen === 'function'`, which stays a property read here. + */ +function countingStore(inner: TripleStore, calls: string[]): TripleStore { + return new Proxy(inner as unknown as object, { + get(target, prop) { + const value = Reflect.get(target, prop); + if (typeof value !== 'function') return value; + return (...args: unknown[]) => { + calls.push(String(prop)); + return (value as (...a: unknown[]) => unknown).apply(target, args); + }; + }, + }) as unknown as TripleStore; +} + +/** Collects the `source` label of every query the sweep issues. */ +function recordQuerySources(store: OxigraphStore): { + sources: string[]; + restore: () => void; +} { + const sources: string[] = []; + const originalQuery = store.query.bind(store); + const spy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + sources.push(options.source); + } + return originalQuery(query, options); + }); + return { sources, restore: () => spy.mockRestore() }; +} + +describe('finalized SWM cleanup sweep gates', () => { + /** + * Reviewer acceptance bullet (a): under sustained load the GC performs no + * discovery and no payload scan. Asserting only that the discovery closure + * was skipped is weaker than the bullet — it leaves room for a refactor to + * hoist a store read (a backlog probe, a resumption-cursor read) above the + * pressure gate. This pins the store itself: the only call permitted while + * pressure is active is reading the pressure snapshot. + */ + it('performs zero store operations and no discovery while the store is under pressure', async () => { + const inner = new OxigraphStore(); + const pressure = installPressureSwitch(inner); + pressure.busy = true; + const calls: string[] = []; + const listContextGraphIds = vi.fn(async () => [CG]); + const listSharedMemoryMetaGraphs = vi.fn(async () => [ + new GraphManager(inner).sharedMemoryMetaUri(CG), + ]); + const service = new FinalizedSwmCleanupService({ + store: countingStore(inner, calls), + writeLocks: new Map>(), + listContextGraphIds, + listSharedMemoryMetaGraphs, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + backlogDepth: 0, + }); + + expect(listContextGraphIds).not.toHaveBeenCalled(); + expect(listSharedMemoryMetaGraphs).not.toHaveBeenCalled(); + expect(calls.filter((name) => name !== 'getPressureSnapshot')).toEqual([]); + expect(calls.length).toBeGreaterThan(0); + }); + + /** + * Pressure that appears *after* the context-graph enumeration resolves — the + * realistic case, since enumeration is the expensive call that tends to run + * just as foreground work arrives. The gate at the top of the context-graph + * loop must abandon the slice before the per-graph meta enumeration. + */ + it('abandons the slice when pressure appears after context-graph enumeration', async () => { + const store = new OxigraphStore(); + const pressure = installPressureSwitch(store); + const listSharedMemoryMetaGraphs = vi.fn(async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ]); + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => { + pressure.busy = true; + return [CG, 'second-cg']; + }, + listSharedMemoryMetaGraphs, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + + expect(listSharedMemoryMetaGraphs).not.toHaveBeenCalled(); + expect(sources).toEqual([]); + restore(); + }); + + /** + * Pressure that appears while a meta graph is being cleaned. The gate after + * `cleanupMetaGraph` must abandon the slice: without it the backlog probe + * runs and throws `StoreSchedulerBusyError` into the worker, converting an + * expected, retryable idle-yield into a recorded sweep failure. + */ + it('abandons the slice when pressure appears during meta-graph cleanup', async () => { + const store = new OxigraphStore(); + const pressure = installPressureSwitch(store); + const sources: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + const result = await originalQuery(query, options); + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + sources.push(options.source); + if (options.source === 'agent.finalizedSwmCleanup.discover') pressure.busy = true; + } + return result; + }); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + + expect(sources).toEqual(['agent.finalizedSwmCleanup.discover']); + expect(sources).not.toContain('agent.finalizedSwmCleanup.backlog'); + querySpy.mockRestore(); + }); + + /** + * A pressure snapshot is a point-in-time sample: a foreground burst can be + * over by the time a callee re-samples it. The gate at the top of the + * meta-graph loop is therefore not redundant with the guard inside + * `cleanupMetaGraph` — it is the only observer of a spike seen at that + * instant, and dropping it lets a slice that already saw pressure go on to + * open a discovery scan. + * + * The spike is one-shot on purpose. Adding a further gate before this one + * keeps the test green (the slice is still abandoned, still with no scan); + * only removing the gate turns it red. + */ + it('abandons the slice on a transient pressure spike seen after meta-graph enumeration', async () => { + const store = new OxigraphStore(); + let spikePending = false; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => { + const busy = spikePending; + spikePending = false; + return { + ackInflight: 0, + healthInflight: 0, + normalInflight: busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }; + }, + }); + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => { + spikePending = true; + return [new GraphManager(store).sharedMemoryMetaUri(CG)]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + + expect(sources).toEqual([]); + restore(); + }); + + /** + * The wall-clock budget is enforced on a caller-supplied clock, so these + * three tests move the clock at each in-loop boundary. `budgetExhausted` is + * what makes the worker reschedule promptly instead of waiting for the + * periodic backstop, so both the flag and the abandoned work are asserted. + */ + it('yields the slice when the budget expires during context-graph enumeration', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const listSharedMemoryMetaGraphs = vi.fn(async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ]); + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => { + clock.now += 60_001; + return [CG, 'second-cg']; + }, + listSharedMemoryMetaGraphs, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + budgetExhausted: true, + deletedItems: 0, + }); + + expect(listSharedMemoryMetaGraphs).not.toHaveBeenCalled(); + expect(sources).toEqual([]); + restore(); + }); + + it('yields the slice when the budget expires during meta-graph enumeration', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const { sources, restore } = recordQuerySources(store); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => { + clock.now += 60_001; + return [new GraphManager(store).sharedMemoryMetaUri(CG)]; + }, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + budgetExhausted: true, + deletedItems: 0, + }); + + // Neither the candidate discovery nor the backlog probe may run once the + // slice is over budget. + expect(sources).toEqual([]); + restore(); + }); + + it('yields the slice when the budget expires inside meta-graph cleanup', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const sources: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + const result = await originalQuery(query, options); + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + sources.push(options.source); + if (options.source === 'agent.finalizedSwmCleanup.discover') clock.now += 60_001; + } + return result; + }); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + budgetExhausted: true, + deletedItems: 0, + }); + + expect(sources).toEqual(['agent.finalizedSwmCleanup.discover']); + expect(sources).not.toContain('agent.finalizedSwmCleanup.backlog'); + querySpy.mockRestore(); + }); + + /** + * End-to-end budget wiring: the worker's prompt-retry decision is driven by a + * `budgetExhausted` produced by the real service, not by a stubbed sweep, and + * a yielded slice must not be miscounted as store pressure. + */ + it('reschedules promptly on a budget-exhausted slice produced by the real service', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const clock = { now: 1_000_000 }; + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + now: () => clock.now, + wallClockBudgetMs: 60_000, + listContextGraphIds: async () => { + clock.now += 60_001; + return [CG]; + }, + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + const timers: Array<{ fn: () => void; delayMs: number }> = []; + const onError = vi.fn(); + const worker = new FinalizedSwmCleanupWorker({ + sweep: () => service.runSweep(), + retryDelayMs: 321, + onError, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + + await worker.runNow(); + + expect(onError).not.toHaveBeenCalled(); + expect(worker.snapshot()).toMatchObject({ runs: 1, pressureSkips: 0, lastError: null }); + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(321); + await worker.close(); + }); + + /** + * Wires the real service into the real worker so a store rejection travels + * the whole path it travels in production. + * + * The snapshot gate is a point-in-time sample and cannot see load that lands + * between the gate and the query, so the background lane reports saturation + * by THROWING instead. `inspectBacklog` even raises that class itself as a + * defensive check straight after the gate, so the service can manufacture the + * error without any scheduler load at all — just the race between the two. + * + * Before this was classified, such a throw skipped the worker's retry + * scheduling entirely (which exists only on the success path) and stranded + * the backlog until the 15-minute backstop. + */ + function schedulerThrowFixture(error: unknown) { + const store = new OxigraphStore(); + installPressureSwitch(store); + const originalQuery = store.query.bind(store); + vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.backlog') throw error; + return originalQuery(query, options); + }); + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [ + new GraphManager(store).sharedMemoryMetaUri(CG), + ], + }); + const timers: Array<{ fn: () => void; delayMs: number }> = []; + const onError = vi.fn(); + const worker = new FinalizedSwmCleanupWorker({ + sweep: () => service.runSweep(), + retryDelayMs: 5_000, + onError, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + return { worker, timers, onError }; + } + + it('retries and counts a scheduler rejection thrown out of the sweep as pressure', async () => { + const { worker, timers, onError } = schedulerThrowFixture( + new StoreSchedulerBusyError('queue_full', 'background', 'agent.finalizedSwmCleanup.backlog'), + ); + + await worker.runNow(); + + // Retried on the prompt cadence rather than left to the periodic backstop. + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(5_000); + // Counted as a deferral, not swallowed as a fault: the counters must stay + // meaningful exactly when deferral is happening. + expect(worker.snapshot()).toMatchObject({ + runs: 1, + pressureSkips: 1, + lastError: null, + backlogStale: true, + }); + expect(onError).not.toHaveBeenCalled(); + await worker.close(); + }); + + /** + * The negative that makes the classification mean something. Broadening it to + * catch every throw would make an unknown fault hot-retry every few seconds + * forever, and the positive test above would still pass — so the boundary + * needs its own assertion. + */ + it('does not retry an unknown sweep failure', async () => { + const { worker, timers, onError } = schedulerThrowFixture(new Error('store exploded')); + + await worker.runNow(); + + expect(timers).toHaveLength(0); + expect(onError).toHaveBeenCalledTimes(1); + expect(worker.snapshot()).toMatchObject({ + runs: 1, + pressureSkips: 0, + lastError: 'store exploded', + }); + await worker.close(); + }); + + /** + * The classification exists at FOUR independent `await` boundaries inside a + * sweep, and a single end-to-end test covers only whichever one it happens to + * trigger — the other three can be deleted with the suite green. Each is + * therefore driven separately here. + * + * The two enumeration boundaries and the two query boundaries also differ in + * what they must preserve: the context-graph enumeration fails before a + * rotation exists, so it defers outright, while the later three must yield + * the rotation and keep the cursor. + */ + type PressureSite = 'context-graph enumeration' | 'meta-graph enumeration' + | 'candidate discovery' | 'the backlog probe'; + + function serviceThrowingAt(site: PressureSite, error: unknown) { + const store = new OxigraphStore(); + installPressureSwitch(store); + const metaGraph = new GraphManager(store).sharedMemoryMetaUri(CG); + const originalQuery = store.query.bind(store); + vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (site === 'candidate discovery' && options?.source === 'agent.finalizedSwmCleanup.discover') { + throw error; + } + if (site === 'the backlog probe' && options?.source === 'agent.finalizedSwmCleanup.backlog') { + throw error; + } + return originalQuery(query, options); + }); + return new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => { + if (site === 'context-graph enumeration') throw error; + return [CG]; + }, + listSharedMemoryMetaGraphs: async () => { + if (site === 'meta-graph enumeration') throw error; + return [metaGraph]; + }, + }); + } + + const PRESSURE_SITES: PressureSite[] = [ + 'context-graph enumeration', + 'meta-graph enumeration', + 'candidate discovery', + 'the backlog probe', + ]; + + it.each(PRESSURE_SITES)( + 'classifies a scheduler rejection raised at %s as a retryable deferral', + async (site) => { + const service = serviceThrowingAt( + site, + new StoreSchedulerBusyError('queue_full', 'background', 'agent.finalizedSwmCleanup'), + ); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + stale: true, + deletedItems: 0, + }); + }, + ); + + it.each(PRESSURE_SITES)( + 'lets an unknown failure at %s stay terminal', + async (site) => { + const service = serviceThrowingAt(site, new Error(`unknown failure at ${site}`)); + + await expect(service.runSweep()).rejects.toThrow(`unknown failure at ${site}`); + }, + ); + + /** + * A converted throw must behave like a pressure yield, not merely return the + * right shape: the context graph it interrupted has to be re-measured whole + * on the next slice. Committing its partial contribution would double-count + * it on resume — the same defect the rotation accumulator exists to prevent, + * arriving through the error path instead of the gate. + */ + it('re-measures an interrupted context graph whole after a converted rejection', async () => { + const store = new OxigraphStore(); + installPressureSwitch(store); + const graphManager = new GraphManager(store); + const rootMeta = graphManager.sharedMemoryMetaUri(CG); + const subMeta = graphManager.sharedMemoryMetaUri(CG, 'named'); + const marker = (label: string, graph: string) => ([ + { subject: `urn:dkg:finalized-swm-cleanup:${label}`, predicate: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', object: 'http://dkg.io/ontology/FinalizedSwmCleanupTask', graph }, + { subject: `urn:dkg:finalized-swm-cleanup:${label}`, predicate: 'http://dkg.io/ontology/finalizedSwmCleanupRoot', object: JSON.stringify(`0x${'11'.repeat(32)}`), graph }, + ]); + await store.insert([...marker('a', rootMeta), ...marker('b', subMeta)]); + + let backlogReads = 0; + const originalQuery = store.query.bind(store); + vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.backlog') { + backlogReads += 1; + // Interrupt AFTER the first meta graph has been measured, so a + // partial contribution exists to be wrongly committed. + if (backlogReads === 2) { + throw new StoreSchedulerBusyError('queue_full', 'background', 'agent.finalizedSwmCleanup.backlog'); + } + } + return originalQuery(query, options); + }); + + const service = new FinalizedSwmCleanupService({ + store, + writeLocks: new Map>(), + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [rootMeta, subMeta], + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: true, + stale: true, + // The one marker measured before the throw is NOT published. + backlogDepth: 0, + }); + + await expect(service.runSweep()).resolves.toMatchObject({ + pressureSkipped: false, + stale: false, + // Two markers total — the interrupted graph re-measured whole, not three. + backlogDepth: 2, + }); + }); +}); diff --git a/packages/agent/test/finalized-swm-cleanup-worker.test.ts b/packages/agent/test/finalized-swm-cleanup-worker.test.ts new file mode 100644 index 0000000000..731438736d --- /dev/null +++ b/packages/agent/test/finalized-swm-cleanup-worker.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, vi } from 'vitest'; +import { FinalizedSwmCleanupWorker } from '../src/finalized-swm-cleanup-worker.js'; + +describe('FinalizedSwmCleanupWorker', () => { + it('records backlog, marker age, pressure skips and deleted items', async () => { + let now = 20_000; + const sweep = vi.fn().mockResolvedValue({ + backlogDepth: 7, + oldestMarkerAt: 5_000, + deletedItems: 2, + pressureSkipped: true, + }); + const worker = new FinalizedSwmCleanupWorker({ + sweep, + now: () => now, + // Keep retry scheduling observable without running another pass. + setTimer: (() => ({ unref() {} })) as never, + clearTimer: () => {}, + }); + + await worker.runNow(); + expect(worker.snapshot()).toMatchObject({ + backlogDepth: 7, + oldestMarkerAgeMs: 15_000, + pressureSkips: 1, + deletedItems: 2, + runs: 1, + lastError: null, + }); + now += 1_000; + expect(worker.snapshot().oldestMarkerAgeMs).toBe(16_000); + await worker.close(); + }); + + /** + * Coalescing has two halves and they need separate evidence. + * + * The re-wake half — a wake arriving during a sweep is not lost — is already + * covered by the yielded-slice test below. The half asserted here is that + * SEVERAL wakes during one sweep collapse to exactly ONE follow-up, which is + * what stops repeated wake-ups creating overlapping store scans. + * + * The previous version of this test closed the worker immediately after + * releasing the sweep, so the follow-up was cancelled before it could be + * observed and the whole in-flight branch could be reduced to a bare `return` + * with this test still green. It now lets the follow-up schedule and fire. + */ + it('collapses several wakes during one sweep into exactly one follow-up sweep', async () => { + const timers: Array<{ fn: () => void; delayMs: number }> = []; + let release!: () => void; + const pending = new Promise((resolve) => { release = resolve; }); + let started = 0; + const sweep = vi.fn(async () => { + started += 1; + // Only the first pass blocks; the follow-up must be free to complete. + if (started === 1) await pending; + return { + backlogDepth: 0, + oldestMarkerAt: null, + deletedItems: 0, + pressureSkipped: false, + }; + }); + const worker = new FinalizedSwmCleanupWorker({ + sweep, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + + expect(worker.wake()).toBeUndefined(); + expect(sweep).not.toHaveBeenCalled(); + timers.shift()!.fn(); + await Promise.resolve(); + expect(sweep).toHaveBeenCalledTimes(1); + + // Three wakes while one sweep is in flight. None may start a sweep, and + // none may schedule a timer of its own. + expect(worker.wake()).toBeUndefined(); + expect(worker.wake()).toBeUndefined(); + expect(worker.wake()).toBeUndefined(); + expect(sweep).toHaveBeenCalledTimes(1); + expect(timers).toHaveLength(0); + + release(); + await new Promise((resolve) => { setImmediate(resolve); }); + + // Exactly one follow-up was scheduled for the three coalesced wakes. + expect(sweep).toHaveBeenCalledTimes(1); + expect(timers).toHaveLength(1); + + timers.shift()!.fn(); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(sweep).toHaveBeenCalledTimes(2); + // A clean second pass asks for nothing further. + expect(timers).toHaveLength(0); + + await worker.close(); + expect(sweep).toHaveBeenCalledTimes(2); + }); + + it('contains sweep errors and exposes the last failure', async () => { + const onError = vi.fn(); + const worker = new FinalizedSwmCleanupWorker({ + sweep: async () => { throw new Error('store unavailable'); }, + onError, + }); + + await expect(worker.runNow()).resolves.toBeUndefined(); + expect(onError).toHaveBeenCalledTimes(1); + expect(worker.snapshot()).toMatchObject({ + runs: 1, + lastError: 'store unavailable', + }); + await worker.close(); + }); + + it('reschedules a yielded wall-clock slice without counting it as pressure', async () => { + const timers: Array<{ fn: () => void; delayMs: number }> = []; + const worker = new FinalizedSwmCleanupWorker({ + sweep: async () => ({ + backlogDepth: 0, + oldestMarkerAt: null, + deletedItems: 0, + pressureSkipped: false, + budgetExhausted: true, + }), + retryDelayMs: 123, + setTimer: ((fn: () => void, delayMs: number) => { + timers.push({ fn, delayMs }); + return { unref() {} }; + }) as never, + clearTimer: () => {}, + }); + + await worker.runNow(); + expect(worker.snapshot().pressureSkips).toBe(0); + expect(timers).toHaveLength(1); + expect(timers[0]!.delayMs).toBe(123); + await worker.close(); + }); +}); diff --git a/packages/agent/test/ka-graph-finalization-handler.test.ts b/packages/agent/test/ka-graph-finalization-handler.test.ts index 4acd8d8b03..ad85b5e183 100644 --- a/packages/agent/test/ka-graph-finalization-handler.test.ts +++ b/packages/agent/test/ka-graph-finalization-handler.test.ts @@ -15,7 +15,9 @@ import { GraphManager, OxigraphStore, StoreSchedulerBusyError, + asGraphWriteGenSource, type Quad, + type TripleStore, } from '@origintrail-official/dkg-storage'; import type { ChainAdapter } from '@origintrail-official/dkg-chain'; import { @@ -24,8 +26,16 @@ import { resolveKnowledgeAssetWorkspaceHead, storeKnowledgeAssetOperationPublicQuads, storeKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, + workspaceOperationSubject, + workspacePublicQuadsDigest, } from '@origintrail-official/dkg-publisher'; -import { FinalizationHandler } from '../src/finalization-handler.js'; +import { + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, + FinalizationHandler, +} from '../src/finalization-handler.js'; import { openSqliteFinalizationRecoveryStore, type SqliteFinalizationRecoveryStore, @@ -37,6 +47,9 @@ import { type ChainReconcilerDeps, } from '../src/chain-reconciler.js'; import { createCursorState } from '../src/reconcile-cursor.js'; +import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped-swm-recovery.js'; +import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; +import { FinalizedSwmCleanupService } from '../src/finalized-swm-cleanup-service.js'; const CG = 'rootless-finalization'; const AUTHOR = '0x1111111111111111111111111111111111111111'; @@ -124,6 +137,33 @@ function legacyFinalizationChain( } as ChainAdapter; } +/** The read the cleanup service takes again once it owns the writer lock. */ +const WORKSPACE_HEAD_READ_PREFIX = 'SELECT ?scopeVersion ?kaUal ?assertionVersion'; + +const WRITE_GEN_CAPABILITY_KEYS = new Set(['getWriteGen', 'innerStore', 'inner']); + +/** + * A store whose per-graph write-generation capability cannot be recovered. + * + * `asGraphWriteGenSource` is documented as fail-open, so on such a store every + * generation comparison in the cleanup service is inert and the head re-read + * taken inside the writer lock is the only remaining proof that nothing moved + * while verification ran outside it. + */ +function withoutWriteGenTracking(inner: OxigraphStore): TripleStore { + return new Proxy(inner as unknown as object, { + get(target, prop) { + if (WRITE_GEN_CAPABILITY_KEYS.has(prop as string)) return undefined; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + has(target, prop) { + if (WRITE_GEN_CAPABILITY_KEYS.has(prop as string)) return false; + return Reflect.has(target, prop); + }, + }) as unknown as TripleStore; +} + async function closeInbox(inbox: SqliteFinalizationRecoveryStore | undefined): Promise { await inbox?.close().catch(() => {}); } @@ -169,13 +209,37 @@ describe('graph-scoped finalization handler', () => { let store: OxigraphStore; let graphManager: GraphManager; let handler: FinalizationHandler; + let writeLocks: Map>; beforeEach(() => { store = new OxigraphStore(); graphManager = new GraphManager(store); - handler = new FinalizationHandler(store, legacyFinalizationChain()); + writeLocks = new Map>(); + handler = new FinalizationHandler(store, legacyFinalizationChain(), { writeLocks }); }); + function cleanupService( + locks: Map> | null = writeLocks, + ): FinalizedSwmCleanupService { + return new FinalizedSwmCleanupService({ + store, + writeLocks: locks ?? undefined, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [graphManager.sharedMemoryMetaUri(CG)], + }); + } + + async function drainFinalizedSwm( + locks: Map> | null = writeLocks, + subGraphName?: string, + ): Promise { + return cleanupService(locks).cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG, subGraphName), + maxCandidates: 16, + }); + } + async function stageGraph(durableAccess?: { accessPolicy: 'ownerOnly' | 'allowList'; allowedPeers?: string[]; @@ -183,6 +247,7 @@ describe('graph-scoped finalization handler', () => { message: FinalizationMessageMsg; swmGraph: string; vmGraph: string; + publicQuads: Quad[]; }> { const scope = createGraphKnowledgeAssetScope(UAL, VERSION); const swmGraph = knowledgeAssetLayerGraphUri( @@ -246,6 +311,7 @@ describe('graph-scoped finalization handler', () => { return { swmGraph, vmGraph, + publicQuads, message: { ual: scope.ual, contextGraphId: CG, @@ -356,6 +422,19 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(vmGraph)).toBe(2); expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toBeUndefined(); + await expect(store.query( + `ASK { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { + ?operation ${JSON.stringify(SHARE_ID)} . + } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); const stale = await store.query( `ASK { GRAPH <${vmGraph}> { ?p ?o } }`, ); @@ -397,6 +476,66 @@ describe('graph-scoped finalization handler', () => { expect(legacyRoots).toMatchObject({ type: 'boolean', value: false }); }); + it('eventually removes a late repeated snapshot after the node becomes idle again', async () => { + const { message, swmGraph, vmGraph, publicQuads } = await stageGraph(); + const metaGraph = graphManager.sharedMemoryMetaUri(CG); + const beforeFinalization = await store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${metaGraph}> { ?s ?p ?o } }`, + ); + expect(beforeFinalization.type).toBe('quads'); + if (beforeFinalization.type !== 'quads') throw new Error('expected SWM metadata'); + const [descriptor] = parseGraphScopedSwmRecoveryDescriptors({ + contextGraphId: CG, + metaQuads: beforeFinalization.quads.map((quad) => ({ ...quad, graph: metaGraph })), + }); + expect(descriptor).toBeDefined(); + if (!descriptor) throw new Error('expected graph-scoped SWM descriptor'); + + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + expect(await store.countQuads(vmGraph)).toBe(publicQuads.length); + + // A delayed/repeated snapshot lands normally. Ingest only re-arms the + // constant-size durable task; it never scans VM or deletes SWM. + const materializer = createSharedMemorySnapshotMaterializer({ + store, + writeLocks, + invalidateListContextGraphsCache: () => {}, + insertReplacementMetadata: (quads) => store.insert([...quads]), + }); + await materializer.withKaWriteLock(CG, undefined, UAL, async () => { + await materializer.ensureFinalizedCleanupTask(CG, descriptor); + await materializer.replaceGraph(swmGraph, publicQuads); + await materializer.replaceHeadMetadata(CG, descriptor); + }); + expect(await store.countQuads(swmGraph)).toBe(publicQuads.length); + expect(await store.countQuads(vmGraph)).toBe(publicQuads.length); + + // Once idle, only the dedicated GC performs the expensive verification + // and exact conditional delete. VM remains byte-for-byte intact. + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + expect(await store.countQuads(vmGraph)).toBe(publicQuads.length); + expect(await drainFinalizedSwm()).toBe(0); + await expect(store.query( + `ASK { GRAPH <${metaGraph}> { ?task ` + + ` ` + + `<${FINALIZED_SWM_CLEANUP_TASK_TYPE}> } }`, + )).resolves.toMatchObject({ type: 'boolean', value: false }); + await expect(store.query( + `ASK { GRAPH <${metaGraph}> { <${workspaceOperationSubject(CG, SHARE_ID)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + const finalVm = await store.query( + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${vmGraph}> { ?s ?p ?o } }`, + ); + expect(finalVm.type).toBe('quads'); + if (finalVm.type !== 'quads') throw new Error('expected VM content'); + expect(workspacePublicQuadsDigest(finalVm.quads.map((quad) => ({ ...quad, graph: '' })))) + .toBe(workspacePublicQuadsDigest(publicQuads.map((quad) => ({ ...quad, graph: '' })))); + }); + it('accepts adapter batch metadata when the singleton KA range matches the UAL', async () => { const { message, vmGraph } = await stageGraph(); @@ -944,7 +1083,8 @@ describe('graph-scoped finalization handler', () => { store.replaceGraphAndSubject = replaceGraphAndSubject; await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); await expect(store.query( `ASK { GRAPH <${metaGraph}> { <${UAL}> "${message.txHash}" } }`, )).resolves.toMatchObject({ type: 'boolean', value: true }); @@ -2135,6 +2275,822 @@ describe('graph-scoped finalization handler', () => { expect(currentHead?.assertionVersion).toBe('2'); }); + it('preserves finalized SWM when the committed VM graph disappears before cleanup', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await store.countQuads(vmGraph)).toBe(2); + + // Simulate external VM loss only after finalization has persisted the + // cleanup marker. The drain must re-verify VM durability and fail closed. + await store.dropGraph(vmGraph); + + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); + }); + + it('preserves finalized SWM when its exact assertion changes before cleanup', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + // Keep the finalized rows but add one new row after the marker is written. + // Exact-digest re-verification must reject this three-row graph. + await store.insert([{ + subject: 'urn:asset:post-finalization-change', + predicate: 'urn:predicate:value', + object: '"newer"', + graph: swmGraph, + }]); + + expect(await store.countQuads(swmGraph)).toBe(3); + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(3); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); + }); + + /** + * Despite its previous name, this test does not observe cleanup serializing + * against a contended lock: the blocking writer is released before the drain + * ever runs. Pointing that writer at an unrelated lock key leaves the test + * green, which is the proof. What it does verify is still worth keeping — + * finalization commits the VM graph even while another writer holds the + * per-KA SWM lock, and it leaves SWM entirely to the independent drain. + * + * The serialization property itself is covered by + * `re-reads the head under the lock when the store cannot track write + * generations`, which queues the drain behind a held lock and asserts the + * ordering across it. + */ + it('commits VM under a held per-KA SWM lock and leaves SWM to the later drain', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + const writeLocks = new Map>(); + const lockingHandler = new FinalizationHandler(store, legacyFinalizationChain(), { + writeLocks, + }); + let releaseWriterLock!: () => void; + let markWriterLockAcquired!: () => void; + const writerLockAcquired = new Promise((resolve) => { + markWriterLockAcquired = resolve; + }); + const holdWriterLock = new Promise((resolve) => { + releaseWriterLock = resolve; + }); + const blocker = withKeyedLocks( + writeLocks, + [swmKaWriteLockKey(CG, undefined, UAL)], + async () => { + markWriterLockAcquired(); + await holdWriterLock; + }, + ); + await writerLockAcquired; + + let markVmCommitted!: () => void; + const vmCommitted = new Promise((resolve) => { + markVmCommitted = resolve; + }); + const replaceGraphAndSubject = store.replaceGraphAndSubject?.bind(store); + if (!replaceGraphAndSubject) throw new Error('Oxigraph replaceGraphAndSubject unavailable'); + store.replaceGraphAndSubject = async ( + graphUri, + quads, + metadataGraph, + subject, + metadata, + options, + ) => { + await replaceGraphAndSubject( + graphUri, + quads, + metadataGraph, + subject, + metadata, + options, + ); + if (graphUri === vmGraph) markVmCommitted(); + }; + + const finalization = lockingHandler.handleFinalizationMessage( + encodeFinalizationMessage(message), + CG, + ); + await vmCommitted; + expect(await store.countQuads(vmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(2); + + releaseWriterLock(); + await blocker; + await finalization; + expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm(writeLocks)).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + }); + + it('preserves an assertion replaced between the pre-lock head read and the in-lock re-check', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await store.countQuads(swmGraph)).toBe(2); + + // Discovery and both payload verifications run OUTSIDE the per-KA writer + // lock by design, so everything the drain has proven is only as fresh as + // the head it read before taking the lock. The re-read taken under the + // lock is the sole check that closes that window: the write-generation + // comparisons are already behind us by then, and the marker ASK only + // proves the cleanup task still exists, not that the assertion is still + // the one that was verified. + // + // Mutating the head *before* the drain does not reach this branch — the + // pre-lock triage in `cleanupMetaGraph` sees the mismatch first and + // retires the task. The race has to land between the two reads. + const originalQuery = store.query.bind(store); + let racedInsideLock = false; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if ( + !racedInsideLock + && options?.source === 'agent.finalizedSwmCleanup' + && query.startsWith(WORKSPACE_HEAD_READ_PREFIX) + ) { + racedInsideLock = true; + // Drop the seam first so the staging writes and the rest of the commit + // path run against the real store. + querySpy.mockRestore(); + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + } + return originalQuery(query, options); + }); + + await expect(drainFinalizedSwm()).resolves.toBe(0); + expect(racedInsideLock).toBe(true); + + // The newer assertion shares the SWM graph URI with the finalized one, so + // a delete here destroys unpublished data rather than a redundant copy. + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(store.query( + `ASK { GRAPH <${swmGraph}> { ` + + ` "newer" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '2' }); + }); + + it('re-reads the head under the lock when the store cannot track write generations', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await store.countQuads(swmGraph)).toBe(2); + + const untrackedStore = withoutWriteGenTracking(store); + expect(asGraphWriteGenSource(untrackedStore)).toBeNull(); + + const lockKey = swmKaWriteLockKey(CG, undefined, UAL); + let releaseWriter!: () => void; + let markWriterHolding!: () => void; + const writerHolding = new Promise((resolve) => { markWriterHolding = resolve; }); + const writerDone = new Promise((resolve) => { releaseWriter = resolve; }); + const blocker = withKeyedLocks(writeLocks, [lockKey], async () => { + markWriterHolding(); + await writerDone; + }); + // `withKeyedLocks` installs its gate synchronously, before its first await. + const writerGate = writeLocks.get(lockKey); + expect(writerGate).toBeDefined(); + await writerHolding; + + const drain = new FinalizedSwmCleanupService({ + store: untrackedStore, + writeLocks, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [graphManager.sharedMemoryMetaUri(CG)], + }).cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + }); + + // Wait until the drain has finished discovery plus both verifications and + // queued behind the writer; the map entry flips to the drain's own gate. + let drainQueuedBehindWriter = false; + for (let attempt = 0; attempt < 1_000 && !drainQueuedBehindWriter; attempt += 1) { + await new Promise((resolve) => { setImmediate(resolve); }); + drainQueuedBehindWriter = writeLocks.get(lockKey) !== writerGate; + } + expect(drainQueuedBehindWriter).toBe(true); + + // The lock holder replaces the assertion the drain just verified. Nothing + // the drain captured before queueing is true any more. + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + releaseWriter(); + await blocker; + + await expect(drain).resolves.toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(store.query( + `ASK { GRAPH <${swmGraph}> { ` + + ` "newer" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '2' }); + }); + + /** + * The lock that matters here belongs to the cleanup SERVICE, not the handler: + * `cleanupService(null)` is what makes this fail closed. The uncoordinated + * handler is incidental — finalization no longer takes a per-KA SWM lock at + * all, and its `writeLocks` option has since been deleted as dead. + */ + it('preserves finalized SWM when the cleanup service has no writer lock map', async () => { + const { message, swmGraph } = await stageGraph(); + const uncoordinated = new FinalizationHandler(store, legacyFinalizationChain()); + + await uncoordinated.handleFinalizationMessage( + encodeFinalizationMessage(message), + CG, + ); + + expect(await cleanupService(null).cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + })).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ shareOperationId: SHARE_ID }); + }); + + /** + * `retireStaleTask` is the ONLY path that removes a cleanup task whose head + * has moved on, and no test reached it — including its call site. Its + * lifecycle transitions are asserted here. + * + * Retirement is deliberately NOT observable through the drain's return value: + * that counts reclaimed SWM lifecycles only, so a retiring sweep reports 0. + * These assert task presence directly for that reason. + */ + async function countCleanupTasks(subGraphName?: string): Promise { + const result = await store.query( + `SELECT (COUNT(DISTINCT ?task) AS ?count) WHERE { GRAPH ` + + `<${graphManager.sharedMemoryMetaUri(CG, subGraphName)}> { ?task ` + + ` ` + + `<${FINALIZED_SWM_CLEANUP_TASK_TYPE}> } }`, + ); + if (result.type !== 'bindings') throw new Error('expected cleanup task count bindings'); + const raw = result.bindings[0]?.['count'] ?? '0'; + return Number.parseInt(raw.replace(/^"|"\^\^.*$/g, ''), 10); + } + + async function deleteWorkspaceHeadSubject(): Promise { + await store.deleteByPattern({ + graph: graphManager.sharedMemoryMetaUri(CG), + subject: `${UAL}#dkg-swm-head`, + }); + } + + it('retires a superseded task without disturbing the newer assertion', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // The head moves to v2 while the task still names v1. + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + + await drainFinalizedSwm(); + + expect(await countCleanupTasks()).toBe(0); + // Asserting only that the stale task went is half a test: a change that + // retires v1 AND damages v2 would pass it. The newer lifecycle must survive + // intact, which is what makes this a lifecycle assertion. + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '2' }); + expect(await store.countQuads(swmGraph)).toBe(2); + await expect(store.query( + `ASK { GRAPH <${swmGraph}> { ` + + ` "newer" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + }); + + it('retires a headless task once its payload is gone', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + await deleteWorkspaceHeadSubject(); + await store.dropGraph(swmGraph); + + await drainFinalizedSwm(); + + expect(await countCleanupTasks()).toBe(0); + }); + + /** + * The guard this whole suite exists for. An absent head with the payload + * still present is NOT a finished lifecycle — it is a resurrected SWM copy + * whose head has not been rebuilt yet. Retiring its task strands that copy + * with nothing left to collect it, which is the resurrection #1996 exists to + * prevent, reached through the retirement path instead of the deletion path. + */ + it('does not retire a headless task while its payload is still present', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // Head gone, payload deliberately left behind. + await deleteWorkspaceHeadSubject(); + expect(await store.countQuads(swmGraph)).toBe(2); + + await drainFinalizedSwm(); + + expect(await countCleanupTasks()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(2); + }); + + /** + * Retirement of a headless task is silently inert when the store cannot + * report write generations — `asGraphWriteGenSource` returns null and the + * absent-head branch returns before its payload check. Blazegraph ships no + * tracker, so this is production behaviour there, and it is indistinguishable + * from "nothing needed retiring" without a test saying otherwise. + * + * Pinning the inertness itself is deliberate: if a later change decides + * retiring on non-write-gen evidence is safe, this goes red and forces that + * to be a considered decision rather than a silent one. + */ + it('leaves a headless task untouched when the store cannot track write generations', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // Exactly the shape that retires in the test above. + await deleteWorkspaceHeadSubject(); + await store.dropGraph(swmGraph); + + const untrackedStore = withoutWriteGenTracking(store); + expect(asGraphWriteGenSource(untrackedStore)).toBeNull(); + await new FinalizedSwmCleanupService({ + store: untrackedStore, + writeLocks, + listContextGraphIds: async () => [CG], + listSharedMemoryMetaGraphs: async () => [graphManager.sharedMemoryMetaUri(CG)], + }).cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + }); + + expect(await countCleanupTasks()).toBe(1); + }); + + /** + * `retireStaleTask`'s own in-lock TOCTOU re-check — the twin of the guard in + * `clearIfStillExact`. The pre-lock triage found a stale head, but by the + * time the writer lock is held the head has moved BACK to matching, so the + * task is live again and must not be retired. Retiring it strands a finalized + * SWM copy with nothing left to collect it. + * + * Narrower than the deletion-path twin — this removes a marker rather than + * payload, so the damage is a stranded copy rather than destroyed data — but + * the same class, and unpinned for the same reason: reaching it needs the + * head to change BETWEEN the two reads, which no static fixture produces. + * + * THE SEAM IS POSITIONAL, NOT BY SOURCE. Both head resolutions on this path + * carry `source: 'agent.finalizedSwmCleanup.discover'`, so the fixture counts + * occurrences: 1st = the pre-lock triage in `cleanupMetaGraph`, 2nd = the + * in-lock re-read. Adding a third query with that source ahead of these + * silently retargets the seam — the test would keep passing while no longer + * exercising the guard. + */ + it('does not retire a task whose head becomes current again inside the lock', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await countCleanupTasks()).toBe(1); + + // Pre-lock triage must see a stale head, so advance it to v2. + await stageNewerWorkspaceAssertion( + swmGraph, + message.privateMerkleRoot, + message.privateTripleCount, + ); + + const originalQuery = store.query.bind(store); + let headReads = 0; + let restoredInsideLock = false; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (query.startsWith(WORKSPACE_HEAD_READ_PREFIX)) { + headReads += 1; + if (headReads === 2 && !restoredInsideLock) { + restoredInsideLock = true; + // Drop the seam first so the restore and the rest of the commit path + // run against the real store. + querySpy.mockRestore(); + // The lifecycle this task names is current again. + await storeKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + shareOperationId: SHARE_ID, + kaUal: UAL, + assertionVersion: '1', + }); + } + } + return originalQuery(query, options); + }); + + await drainFinalizedSwm(); + + expect(restoredInsideLock).toBe(true); + // Still armed: the task describes a lifecycle that is live again. + expect(await countCleanupTasks()).toBe(1); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toMatchObject({ assertionVersion: '1', shareOperationId: SHARE_ID }); + }); + + it('defers durable cleanup while the store is busy and resumes after restart when idle', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + let busy = true; + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + + busy = false; + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + })).resolves.toBeUndefined(); + const immutableFinalizationTombstone = await store.query( + `ASK { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { ` + + `<${workspaceOperationSubject(CG, SHARE_ID)}> ` + + `<${FINALIZED_SWM_CLEANUP_ROOT_PREDICATE}> ?root } }`, + ); + expect(immutableFinalizationTombstone).toMatchObject({ + type: 'boolean', + value: true, + }); + }); + + it('never bypasses active store pressure for finalized cleanup', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + Object.defineProperty(store, 'getPressureSnapshot', { + configurable: true, + value: () => ({ + ackInflight: 1, + healthInflight: 1, + normalInflight: 2, + backgroundInflight: 2, + ackQueued: 3, + healthQueued: 2, + normalQueued: 4, + backgroundQueued: 4, + maxConcurrent: 4, + ackReservedSlots: 1, + }), + }); + + const querySpy = vi.spyOn(store, 'query'); + expect(await drainFinalizedSwm()).toBe(0); + expect(await store.countQuads(swmGraph)).toBe(2); + expect(querySpy).not.toHaveBeenCalled(); + querySpy.mockRestore(); + }); + + it('discovers the independent cleanup task for bounded subgraph cleanup', async () => { + const subGraphName = 'named-cleanup'; + const { message, swmGraph } = await stageGraph(undefined, subGraphName); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + const discoverQueries: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.discover') { + discoverQueries.push(query); + } + return originalQuery(query, options); + }); + + await expect(cleanupService().cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG, subGraphName), + maxCandidates: 1, + })).resolves.toBe(1); + expect(discoverQueries.some((query) => query.includes( + ' ', + ))).toBe(true); + expect(await store.countQuads(swmGraph)).toBe(0); + await expect(resolveKnowledgeAssetWorkspaceHead({ + store, + graphManager, + contextGraphId: CG, + kaUal: UAL, + subGraphName, + })).resolves.toBeUndefined(); + querySpy.mockRestore(); + }); + + it('keeps cleanup background-only and leaves retry to the worker after scheduler pressure', async () => { + const { message, swmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + + const originalQuery = store.query.bind(store); + let injectedBusyTimeout = false; + const cleanupPriorities: Array = []; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source?.startsWith('agent.finalizedSwmCleanup')) { + cleanupPriorities.push(options.priority); + } + if ( + !injectedBusyTimeout + && options?.source === 'agent.finalizedSwmCleanup.discover' + && query.includes('SELECT DISTINCT ?task ?ual ?version ?root ?shareId') + ) { + injectedBusyTimeout = true; + throw new StoreSchedulerBusyError( + 'queue_wait_timeout', + 'background', + options.source, + ); + } + return originalQuery(query, options); + }); + + await expect(cleanupService().cleanupKnownMetaGraph({ + contextGraphId: CG, + swmMetaGraph: graphManager.sharedMemoryMetaUri(CG), + maxCandidates: 16, + })).rejects.toBeInstanceOf(StoreSchedulerBusyError); + expect(injectedBusyTimeout).toBe(true); + expect(cleanupPriorities.length).toBeGreaterThan(0); + expect(new Set(cleanupPriorities)).toEqual(new Set(['background'])); + expect(await store.countQuads(swmGraph)).toBe(2); + querySpy.mockRestore(); + }); + + it('repairs VM from the immutable operation snapshot after deferred SWM cleanup', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + const internals = handler as unknown as { + verifyChainCgBinding: (kaId: bigint, cgId: string) => Promise; + }; + internals.verifyChainCgBinding = async () => true; + + await store.dropGraph(vmGraph); + await store.deleteByPattern({ + graph: graphManager.metaGraphUri(CG), + subject: UAL, + }); + + await expect(handler.handleChainReconciledKC({ + contextGraphId: CG, + onChainCgId: '42', + ual: UAL, + merkleRoot: message.kcMerkleRoot, + publisherAddress: PUBLISHER, + kaId: PACKED_KA_ID, + versionBlock: 123, + authorAddress: AUTHOR, + trustedAssertionEvidence: trustedRecoveryEvidence(message), + }, createOperationContext('system'))).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + expect(await store.countQuads(swmGraph)).toBe(0); + }); + + /** + * Stage extra operation subjects for the SAME assertion, so late-receipt + * discovery has several candidates to choose between. `shareOperationId` + * values are chosen to sort BEFORE the real one, because discovery is + * `ORDER BY ?shareId` and that order is unrelated to which candidate matches. + */ + async function stageDecoyOperations(input: { + count: number; + prefix: string; + contentFor: (index: number) => string; + }): Promise { + const scope = createGraphKnowledgeAssetScope(UAL, VERSION); + const shareIds: string[] = []; + for (let index = 0; index < input.count; index += 1) { + const shareOperationId = `${input.prefix}-${index}`; + shareIds.push(shareOperationId); + const value = input.contentFor(index); + await storeKnowledgeAssetOperationPublicQuads({ + store, + graphManager, + contextGraphId: CG, + shareOperationId, + kaUal: scope.ual, + assertionVersion: scope.assertionVersion, + // Same public triple count as the real assertion, so the count filter + // in discovery cannot separate them — only a full payload resolution can. + quads: [ + { subject: 'urn:asset:one', predicate: 'urn:predicate:value', object: `"${value}-one"`, graph: '' }, + { subject: 'urn:asset:two', predicate: 'urn:predicate:value', object: `"${value}-two"`, graph: '' }, + ], + publisherPeerId: '12D3KooWPublisher', + }); + } + return shareIds; + } + + async function reconcileFromImmutableSnapshot( + message: FinalizationMessageMsg, + ): Promise { + const internals = handler as unknown as { + verifyChainCgBinding: (kaId: bigint, cgId: string) => Promise; + }; + internals.verifyChainCgBinding = async () => true; + return handler.handleChainReconciledKC({ + contextGraphId: CG, + onChainCgId: '42', + ual: UAL, + merkleRoot: message.kcMerkleRoot, + publisherAddress: PUBLISHER, + kaId: PACKED_KA_ID, + versionBlock: 123, + authorAddress: AUTHOR, + // Carries no publicQuadsDigest, so discovery cannot pre-filter by content + // and every candidate must be resolved to be told apart. This is the case + // a candidate cap silently broke. + trustedAssertionEvidence: trustedRecoveryEvidence(message), + }, createOperationContext('system')); + } + + /** + * Regression guard against re-introducing a candidate cap. + * + * Discovery orders by `?shareId`, which has no relationship to which snapshot + * actually matches the receipt. Truncating that list therefore does not do + * less work, it returns a different answer — and because the list is + * deterministic, every retry re-derives the identical truncation, so the + * receipt is stranded permanently rather than delayed. Five candidates with + * the matching one last is past the boundary of the cap this replaced. + */ + it('verifies a late receipt whose matching snapshot sorts last among many candidates', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + + // 'aaa-decoy-*' sorts before 'graph-finalization-share'. + await stageDecoyOperations({ + count: 4, + prefix: 'aaa-decoy', + contentFor: (index) => `decoy-${index}`, + }); + await store.dropGraph(vmGraph); + await store.deleteByPattern({ graph: graphManager.metaGraphUri(CG), subject: UAL }); + + await expect(reconcileFromImmutableSnapshot(message)).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + }); + + /** + * The subtle half of the content memo: a THROW must not memo. + * + * Failing to READ one operation's snapshot says nothing about whether that + * content is correct, and a sibling operation may hold a readable copy of the + * very same content. Memoing on throw would therefore skip the readable copy + * and strand the receipt. Both candidates here carry identical content, so + * they share a digest — which is exactly when a wrongly-placed memo bites. + */ + it('still verifies a readable snapshot after an identical-content sibling fails to resolve', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + + // Same content as the real assertion, so it advertises the same digest, and + // it sorts first. Its snapshot payload is then made unreadable. + const scope = createGraphKnowledgeAssetScope(UAL, VERSION); + await storeKnowledgeAssetOperationPublicQuads({ + store, + graphManager, + contextGraphId: CG, + shareOperationId: 'aaa-unreadable-twin', + kaUal: scope.ual, + assertionVersion: scope.assertionVersion, + quads: [ + { subject: 'urn:asset:one', predicate: 'urn:predicate:value', object: '"one"', graph: '' }, + { subject: 'urn:asset:two', predicate: 'urn:predicate:value', object: '"two"', graph: '' }, + ], + privateMerkleRoot: message.privateMerkleRoot, + privateTripleCount: message.privateTripleCount, + publisherPeerId: '12D3KooWPublisher', + }); + const twinSnapshotGraph = (await store.query( + `SELECT ?graph WHERE { GRAPH <${graphManager.sharedMemoryMetaUri(CG)}> { ` + + `<${workspaceOperationSubject(CG, 'aaa-unreadable-twin')}> ` + + ` ?graph } }`, + )); + if (twinSnapshotGraph.type !== 'bindings' || !twinSnapshotGraph.bindings[0]?.['graph']) { + throw new Error('expected a snapshot graph for the twin operation'); + } + await store.dropGraph(twinSnapshotGraph.bindings[0]['graph']!); + + await store.dropGraph(vmGraph); + await store.deleteByPattern({ graph: graphManager.metaGraphUri(CG), subject: UAL }); + + await expect(reconcileFromImmutableSnapshot(message)).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + }); + + /** + * The other half of the memo: a content rejection MUST memo, or the bound on + * repeated work does not exist and we are back to a full payload read per + * candidate. The answer is identical either way, so this can only be observed + * by counting payload resolutions — an output-equality assertion here would + * pass with the memo deleted. + */ + it('resolves an already-rejected snapshot digest only once across candidates', async () => { + const { message, swmGraph, vmGraph } = await stageGraph(); + await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); + + // Two candidates with IDENTICAL wrong content, so they advertise the same + // digest and the second is provably a repeat of work already done. + await stageDecoyOperations({ count: 2, prefix: 'aaa-dup', contentFor: () => 'same-wrong' }); + await store.dropGraph(vmGraph); + await store.deleteByPattern({ graph: graphManager.metaGraphUri(CG), subject: UAL }); + + const payloadReads: string[] = []; + const originalQuery = store.query.bind(store); + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if ( + options?.source === 'agent.finalization.resolveImmutableSnapshotPayload' + && query.startsWith('CONSTRUCT') + ) payloadReads.push(query); + return originalQuery(query, options); + }); + + await expect(reconcileFromImmutableSnapshot(message)).resolves.toBe('promoted'); + expect(await store.countQuads(vmGraph)).toBe(2); + // One rejected duplicate plus the matching snapshot — never the skipped twin. + expect(payloadReads).toHaveLength(2); + querySpy.mockRestore(); + }); + it('rejects mixed graph-scope and legacy-root finalization envelopes', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage( @@ -2172,7 +3128,7 @@ describe('graph-scoped finalization handler', () => { expect(await store.countQuads(swmGraph)).toBe(2); }); - it('verifies chain binding and exact private VM metadata without deleting unverified SWM', async () => { + it('verifies chain binding and exact private VM metadata before cleaning an exact SWM copy', async () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); @@ -2217,7 +3173,8 @@ describe('graph-scoped finalization handler', () => { }, createOperationContext('system'))).resolves.toBe('already-confirmed'); expect(bindingVerified).toBe(true); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); }); it('recognizes only exact confirmed Verifiable Memory metadata after the workspace head is lost', async () => { @@ -2498,7 +3455,8 @@ describe('graph-scoped finalization handler', () => { const { message, swmGraph, vmGraph } = await stageGraph(); await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); expect(await store.countQuads(vmGraph)).toBe(2); - expect(await store.countQuads(swmGraph)).toBe(2); + expect(await drainFinalizedSwm()).toBe(1); + expect(await store.countQuads(swmGraph)).toBe(0); const metaGraph = `did:dkg:context-graph:${CG}/_meta`; const materializedVersionPredicate = 'http://dkg.io/ontology/materializedVersion'; @@ -2615,9 +3573,10 @@ describe('graph-scoped finalization handler', () => { await handler.handleFinalizationMessage(encodeFinalizationMessage(message), CG); const staged = await store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${swmGraph}> { ?s ?p ?o } }`, + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${vmGraph}> { ?s ?p ?o } }`, ); if (staged.type !== 'quads') throw new Error('expected staged SWM quads'); + await store.insert(staged.quads.map((quad) => ({ ...quad, graph: swmGraph }))); await storeKnowledgeAssetOperationPublicQuads({ store, graphManager, @@ -2729,9 +3688,10 @@ describe('graph-scoped finalization handler', () => { await store.deleteByPattern({ graph: metaGraph, subject: UAL }); const staged = await store.query( - `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${swmGraph}> { ?s ?p ?o } }`, + `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${vmGraph}> { ?s ?p ?o } }`, ); if (staged.type !== 'quads') throw new Error('expected staged SWM quads'); + await store.insert(staged.quads.map((quad) => ({ ...quad, graph: swmGraph }))); await storeKnowledgeAssetOperationPublicQuads({ store, graphManager, diff --git a/packages/agent/test/swm-public-snapshot-materialization.test.ts b/packages/agent/test/swm-public-snapshot-materialization.test.ts index 276e279440..e9e3f9bc72 100644 --- a/packages/agent/test/swm-public-snapshot-materialization.test.ts +++ b/packages/agent/test/swm-public-snapshot-materialization.test.ts @@ -16,8 +16,8 @@ * 4. an already-materialized asset is left alone * 5. a failed replace keeps the phase incomplete and WITHHOLDS the meta * insert, so a marker can never certify a graph that was not written - * 6. after a replace, the stale head metadata is swapped out BEFORE the - * fresh verified meta is appended (graph → head swap → meta ordering) + * 6. after a graph replace, stale head metadata is replaced with the fresh + * verified rows inside the same per-KA lock (graph → locked metadata swap) * 7. a KA under a REGISTERED subgraph is materialized too — dropping the * subgraph admission pass-through must fail these tests * 8. a node with NO cached snapshot fetches it over the network @@ -184,6 +184,9 @@ function harness(overrides: HarnessOverrides = {}) { overrides.onLockRequested?.(); return withKeyedLocks(lockMap, [swmKaWriteLockKey(contextGraphId, subGraphName, kaUal)], fn); }, + ensureFinalizedCleanupTask: async () => { + events.push('cleanup-task-ensured'); + }, isGraphAssetMaterialized: async () => { events.push('content-checked'); return overrides.contentPresent?.() ?? false; @@ -200,6 +203,8 @@ function harness(overrides: HarnessOverrides = {}) { replaceHeadMetadata: async (contextGraphId, descriptor) => { events.push('head-swapped'); headSwaps.push({ contextGraphId, headSubject: descriptor.headSubject }); + inserted.push([...descriptor.metadataQuads]); + events.push('meta-replaced'); }, }, publicSnapshotStore: snapshotStore, @@ -229,18 +234,16 @@ describe('public SWM snapshot materialization', () => { expect(h.inserted.some((batch) => batch.some((q) => q.graph === WS_META))).toBe(true); }); - it('swaps the stale head metadata after the replace and BEFORE the meta append', async () => { - // The meta insert below is append/union-style: without the head swap the - // old and new version rows stack on one subject and LIMIT-1 readers - // (`resolveKnowledgeAssetWorkspaceHead`) can return either — the stale - // head bug. Ordering matters both ways: graph before swap (a crash leaves - // repairable content, never a head without content), swap before append - // (the fresh rows land on a clean subject). + it('replaces stale head metadata inside the locked swap after replacing the graph', async () => { + // The verified graph-scoped metadata must land in the same locked step as + // the delete. Appending it later would let a live write slip between the + // two operations and recreate a multi-version head. const h = harness({ storedHead: () => ({ version: '1', needsRepair: false }), contentPresent: () => false }); await h.run(); expect(h.events.indexOf('replaced')).toBeGreaterThan(-1); expect(h.events.indexOf('head-swapped')).toBeGreaterThan(h.events.indexOf('replaced')); - expect(h.events.indexOf('meta-inserted')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events.indexOf('meta-replaced')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events).not.toContain('meta-inserted'); expect(h.headSwaps).toEqual([{ contextGraphId: CG, headSubject: `${UAL}#dkg-swm-head` }]); }); @@ -285,6 +288,7 @@ describe('public SWM snapshot materialization', () => { expect(h.events.indexOf('version-read')).toBeGreaterThan(h.events.indexOf('gossip-committed')); expect(h.events).not.toContain('replaced'); expect(h.events).not.toContain('head-swapped'); + expect(h.inserted.every((batch) => batch.every((q) => q.graph !== WS_META))).toBe(true); expect(summary.failedPhases).toBe(0); }); @@ -307,6 +311,22 @@ describe('public SWM snapshot materialization', () => { expect(summary.failedPhases).toBe(0); }); + it('does not perform finalized cleanup while materializing a late snapshot', async () => { + // Eventual convergence deliberately permits a temporary SWM copy. The + // snapshot path performs its normal write only; the real store-backed + // metadata replacement re-arms the durable task for the independent GC. + const h = harness({ + storedHead: () => ({ version: '1', needsRepair: false }), + contentPresent: () => false, + }); + const summary = await h.run(); + expect(h.events).toContain('version-read'); + expect(h.events).toContain('content-checked'); + expect(h.events).toContain('replaced'); + expect(h.events).toContain('head-swapped'); + expect(summary.failedPhases).toBe(0); + }); + it('collapses union-insert residue on the skip path when the head needs repair', async () => { // Content already matches the descriptor, but the head subject carries // several version/operation rows (e.g. a prior round failed between the @@ -316,7 +336,8 @@ describe('public SWM snapshot materialization', () => { const summary = await h.run(); expect(h.events).not.toContain('replaced'); expect(h.events).toContain('head-swapped'); - expect(h.events.indexOf('meta-inserted')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events.indexOf('meta-replaced')).toBeGreaterThan(h.events.indexOf('head-swapped')); + expect(h.events).not.toContain('meta-inserted'); expect(summary.failedPhases).toBe(0); }); diff --git a/packages/agent/test/swm-snapshot-materializer.test.ts b/packages/agent/test/swm-snapshot-materializer.test.ts index 381a1a432f..39de9430fa 100644 --- a/packages/agent/test/swm-snapshot-materializer.test.ts +++ b/packages/agent/test/swm-snapshot-materializer.test.ts @@ -31,22 +31,36 @@ import { type OperationContext, } from '@origintrail-official/dkg-core'; import { + computeFlatKCRootV10, generateKnowledgeAssetShareMetadata, resolveKnowledgeAssetWorkspaceHead, + swmKaWriteLockKey, + withKeyedLocks, workspacePublicQuadsDigest, + type KnowledgeAssetWorkspaceHead, type WorkspacePublicSnapshotStore, } from '@origintrail-official/dkg-publisher'; import { GraphManager, OxigraphStore, type Quad, type TripleStore } from '@origintrail-official/dkg-storage'; import { parseGraphScopedSwmRecoveryDescriptors } from '../src/sync/graph-scoped-swm-recovery.js'; -import { createSharedMemorySnapshotMaterializer } from '../src/sync/requester/swm-snapshot-materializer.js'; +import { + createSharedMemorySnapshotMaterializer, + replaceGraphScopedSwmHeadMetadata, +} from '../src/sync/requester/swm-snapshot-materializer.js'; import { runSharedMemorySync } from '../src/sync/requester/shared-memory-sync.js'; import type { SyncPageResult } from '../src/sync/requester/page-fetch.js'; +import { + FinalizationHandler, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + FINALIZED_SWM_CLEANUP_TASK_TYPE, +} from '../src/finalization-handler.js'; const CG = 'ws00-materializer-real-store'; const WS_META = contextGraphWorkspaceMetaGraphUri(CG); const DKG = 'http://dkg.io/ontology/'; const XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer'; const UAL = 'did:dkg:hardhat:31337/0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/9'; +const FINALIZED_ROOT = `"0x${'11'.repeat(32)}"`; const ctx: OperationContext = { operationId: 'test', operationName: 'sync' } as never; class MemorySnapshotStore implements WorkspacePublicSnapshotStore { @@ -116,6 +130,7 @@ function materializerFor(store: TripleStore) { store, writeLocks: new Map>(), invalidateListContextGraphsCache: () => { invalidations += 1; }, + insertReplacementMetadata: (quads) => store.insert([...quads]), }); return { materializer, invalidations: () => invalidations }; } @@ -132,6 +147,19 @@ async function distinctObjects(store: TripleStore, graph: string, subject: strin return result.bindings.map((row) => String(row['o'])).sort(); } +async function distinctSubjects( + store: TripleStore, + graph: string, + predicate: string, + object: string, +): Promise { + const result = await store.query( + `SELECT DISTINCT ?s WHERE { GRAPH <${graph}> { ?s <${predicate}> <${object}> } }`, + ); + if (result.type !== 'bindings') throw new Error(`unexpected ${result.type}`); + return result.bindings.map((row) => String(row['s'])).sort(); +} + describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', () => { it('shares one assertion graph across versions (the premise of the digest guard)', () => { expect(v1.assertionGraph).toBe(v2.assertionGraph); @@ -199,7 +227,7 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', }); describe('replaceHeadMetadata', () => { - it('deletes the head and every referenced operation, sparing unrelated subjects', async () => { + it('replaces the head and every referenced operation, sparing unrelated subjects', async () => { const store = new OxigraphStore(); await store.insert([...v1.meta]); await store.insert([...v2.meta]); @@ -214,9 +242,11 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', await materializer.replaceHeadMetadata(CG, descriptorFor(v2)); - expect(await distinctObjects(store, WS_META, v2.headSubject, `${DKG}assertionVersion`)).toEqual([]); + expect(await distinctObjects(store, WS_META, v2.headSubject, `${DKG}assertionVersion`)) + .toEqual([`"2"^^<${XSD_INTEGER}>`]); expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)).toEqual([]); - expect(await distinctObjects(store, WS_META, v2.operationSubject, `${DKG}shareOperationId`)).toEqual([]); + expect(await distinctObjects(store, WS_META, v2.operationSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v2.operationId}"`]); expect(await distinctObjects(store, WS_META, unrelated.subject, `${DKG}shareOperationId`)).toEqual(['"unrelated"']); }); @@ -237,10 +267,121 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', await materializer.replaceHeadMetadata(CG, descriptorFor(v1)); - expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)).toEqual([]); - expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)).toEqual([]); + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v1.operationId}"`]); + expect(await distinctObjects(store, WS_META, v1.operationSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v1.operationId}"`]); expect(await distinctObjects(store, WS_META, foreignOp, `${DKG}shareOperationId`)).toEqual(['"foreign-op"']); }); + + it('preserves the operation tombstone and re-arms GC for the exact lifecycle', async () => { + const store = new OxigraphStore(); + await store.insert([ + ...v1.meta, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: '"2026-07-31T10:00:00.000Z"^^', + graph: WS_META, + }, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: '"2026-07-31T11:00:00.000Z"^^', + graph: WS_META, + }, + ]); + const { materializer } = materializerFor(store); + + await materializer.withKaWriteLock(CG, undefined, UAL, async () => { + const descriptor = descriptorFor(v1); + await materializer.ensureFinalizedCleanupTask(CG, descriptor); + await materializer.replaceHeadMetadata(CG, descriptor); + }); + + expect(await distinctObjects( + store, + WS_META, + v1.headSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([FINALIZED_ROOT]); + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + )).toEqual([ + '"2026-07-31T10:00:00Z"^^', + ]); + const tasks = await distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + ); + expect(tasks).toHaveLength(1); + expect(await distinctObjects(store, WS_META, tasks[0]!, `${DKG}kaUal`)).toEqual([UAL]); + expect(await distinctObjects( + store, + WS_META, + tasks[0]!, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([FINALIZED_ROOT]); + }); + + it('drops the old finalized-cleanup token when synchronized metadata is a newer lifecycle', async () => { + const store = new OxigraphStore(); + await store.insert([ + ...v1.meta, + { + subject: v1.headSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + ]); + const { materializer } = materializerFor(store); + + await materializer.withKaWriteLock(CG, undefined, UAL, () => + materializer.replaceHeadMetadata(CG, descriptorFor(v2))); + + expect(await distinctObjects( + store, + WS_META, + v2.headSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + expect(await distinctObjects( + store, + WS_META, + v2.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toEqual([]); + }); }); it('replaceGraph writes atomically and invalidates the list cache', async () => { @@ -362,5 +503,294 @@ describe('createSharedMemorySnapshotMaterializer against a real OxigraphStore', expect(again.failedPhases).toBe(0); expect(h.replaceCalls()).toBe(1); }); + + it('lets a late snapshot restore temporarily and re-arms the independent GC task', async () => { + const store = new OxigraphStore(); + await store.insert([ + ...v1.meta, + { + subject: v1.operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + ]); + const h = realHarness(store, v1); + + const first = await h.run(); + expect(first.failedPhases).toBe(0); + expect(h.replaceCalls()).toBe(1); + const { materializer } = materializerFor(store); + expect(await materializer.isGraphAssetMaterialized(descriptorFor(v1))).toBe(true); + expect(await distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + )).toHaveLength(1); + + // A repeated snapshot is a no-op because content now matches; it does not + // create duplicate tasks or add any cleanup work to the sync path. + const again = await h.run(); + expect(again.failedPhases).toBe(0); + expect(h.replaceCalls()).toBe(1); + expect(await distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + )).toHaveLength(1); + }); + + /** + * The re-arm invariant, pinned on EVERY decision path — not just the two a + * happy-path catch-up happens to take. + * + * `runSharedMemorySync` reaches four distinct outcomes inside the per-KA + * write lock, and only two of them call `replaceHeadMetadata` (which re-arms + * the GC task from the tombstone read it already performs). The other two + * write nothing at all and must re-arm via `ensureFinalizedCleanupTask`. + * Covering only the writing paths lets the whole gate be removed — the ASK + * can be forced to `false`, or either explicit call deleted — while a + * happy-path suite stays green and a finalized SWM copy is resurrected with + * no GC task to collect it. That is the exact resurrection this work exists + * to prevent, so each path is asserted on its own. + * + * The negative half matters just as much: a KA that was never finalized must + * arm NOTHING, or the GC gains a backlog entry for a live asset. + */ + describe('re-arms the finalized-cleanup GC task on every decision path', () => { + const tombstone = (operationSubject: string): Quad[] => [ + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + object: FINALIZED_ROOT, + graph: WS_META, + }, + { + subject: operationSubject, + predicate: FINALIZED_SWM_CLEANUP_MARKED_AT_PREDICATE, + object: '"2026-07-31T10:00:00.000Z"^^', + graph: WS_META, + }, + ]; + + const armedTasks = (store: TripleStore): Promise => distinctSubjects( + store, + WS_META, + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', + FINALIZED_SWM_CLEANUP_TASK_TYPE, + ); + + /** + * Each seed steers the lock body to exactly ONE outcome for a served v1 + * snapshot. Keep them distinguishable: if two seeds collapsed onto the same + * branch the suite would look complete while leaving a path unguarded. + */ + const paths: Record Promise> = { + // Stored head is NEWER than the descriptor: sync must not write. + 'superseded head': async (store) => { + await store.insert([...v2.meta]); + }, + // Content already present, head rows absent: repair via replaceHeadMetadata. + 'materialized with a missing head': async (store) => { + await store.insert([ + ...v1.meta.filter((quad) => quad.subject !== v1.headSubject), + ...inGraph(v1.payload, v1.assertionGraph), + ]); + }, + // Content present and head already clean: sync must not write. + 'materialized with a clean head': async (store) => { + await store.insert([...v1.meta, ...inGraph(v1.payload, v1.assertionGraph)]); + }, + // Content absent: full replaceGraph + replaceHeadMetadata. + 'absent content': async (store) => { + await store.insert([...v1.meta]); + }, + }; + + for (const [label, seed] of Object.entries(paths)) { + it(`arms exactly one task for a finalized operation: ${label}`, async () => { + const store = new OxigraphStore(); + await seed(store); + await store.insert(tombstone(v1.operationSubject)); + + const summary = await realHarness(store, v1).run(); + + expect(summary.failedPhases).toBe(0); + const tasks = await armedTasks(store); + expect(tasks).toHaveLength(1); + // Bound to THIS lifecycle: an arbitrary task subject would satisfy a + // bare length check while pointing the GC at the wrong asset. + expect(await distinctObjects(store, WS_META, tasks[0]!, `${DKG}kaUal`)).toEqual([UAL]); + }); + + it(`arms nothing when the operation was never finalized: ${label}`, async () => { + const store = new OxigraphStore(); + await seed(store); + + const summary = await realHarness(store, v1).run(); + + expect(summary.failedPhases).toBe(0); + expect(await armedTasks(store)).toEqual([]); + }); + } + }); + }); + + /** + * The cleanup-marker write must serialize against catch-up's replacement of + * the SAME operation subject. + * + * `replaceGraphScopedSwmHeadMetadata` reads the tombstone, deletes the + * operation subject, then re-inserts from that snapshot. A marker written + * inside that window is destroyed by the delete and never restored, because + * the snapshot predates it. The independent task subject survives, so the GC + * cleans the lifecycle once and retires the task — and the next catch-up + * re-materializes the SWM copy with no tombstone left to re-arm cleanup from. + * The finalized copy is then resurrected permanently, which is the exact + * failure this component exists to prevent. + * + * The interleaving is injected rather than raced: a marker write that only + * *sometimes* lands in the window is not a proof, and this is precisely the + * race that passes ninety-nine runs in a hundred. The marker is started at + * the moment catch-up is about to delete the operation subject and given + * twenty event-loop turns to complete. Unlocked it finishes inside that + * window every time; correctly locked it CANNOT finish, because the lock is + * held by the replace itself — so the assertion turns on mutual exclusion, + * not on timing. + */ + it('preserves the tombstone when a cleanup marker lands during a head replace', async () => { + const store = new OxigraphStore(); + await store.insert([...v1.meta]); + const writeLocks = new Map>(); + const handler = new FinalizationHandler(store, undefined as never, { writeLocks }); + const scope = createGraphKnowledgeAssetScope(UAL, 1); + const expectedHead = { + kaUal: UAL, + assertionVersion: '1', + assertionGraph: v1.assertionGraph, + publicQuadsDigest: v1.digest, + publicTripleCount: v1.payload.length, + privateTripleCount: 0, + shareOperationId: v1.operationId, + publisherPeerId: 'peer-source', + allowedPeers: [], + } as unknown as KnowledgeAssetWorkspaceHead; + + let markerStarted = false; + let marker: Promise | undefined; + const realDeleteByPattern = store.deleteByPattern.bind(store); + store.deleteByPattern = (async (pattern: Partial, options?: unknown) => { + if (!markerStarted && pattern.subject === v1.operationSubject) { + markerStarted = true; + marker = (handler as unknown as { + markFinalizedGraphScopedSwmForCleanup(input: unknown): Promise; + }).markFinalizedGraphScopedSwmForCleanup({ + contextGraphId: CG, + scope, + expectedHead, + expectedMerkleRoot: new Uint8Array(32).fill(0xab), + ctx, + }); + for (let turn = 0; turn < 20; turn += 1) { + await new Promise((resolve) => { setImmediate(resolve); }); + } + } + return realDeleteByPattern(pattern, options as never); + }) as typeof store.deleteByPattern; + + await withKeyedLocks(writeLocks, [swmKaWriteLockKey(CG, undefined, UAL)], () => + replaceGraphScopedSwmHeadMetadata({ + store, + contextGraphId: CG, + descriptor: descriptorFor(v1), + sourcePrefix: 'test.replaceHead', + insertReplacementMetadata: (quads) => store.insert([...quads]), + })); + await marker; + + expect(markerStarted).toBe(true); + // The tombstone the GC re-arms from must have survived the replace. + expect(await distinctObjects( + store, + WS_META, + v1.operationSubject, + FINALIZED_SWM_CLEANUP_ROOT_PREDICATE, + )).toHaveLength(1); + // And the lifecycle itself is still intact, so this is not passing because + // the replace silently did nothing. + expect(await distinctObjects(store, WS_META, v1.headSubject, `${DKG}shareOperationId`)) + .toEqual([`"${v1.operationId}"`]); + }); + + /** + * Immutable-snapshot discovery must not let `LIMIT` truncate the CANDIDATE + * SET — only the work. + * + * The discovery query binds every same-version operation and takes + * `ORDER BY ?shareId LIMIT 16`. While the triple-count and digest tests ran + * in JS on the returned rows, sixteen non-matching operations sorting ahead + * of the real one filled the window and the matching snapshot was never + * examined. The ordering is deterministic over stable store state, so this is + * not a flake that clears on retry: every attempt re-derives the identical + * sixteen and strands the same recovery permanently. + * + * This lives with the materializer suite rather than the finalization suite + * on purpose: it is driven directly against the private method with a real + * store, needs no chain fixture, and keeps this branch out of a test file + * another branch is editing. + */ + it('finds a matching snapshot that sorts past the discovery limit', async () => { + const store = new OxigraphStore(); + const snapshotStore = new MemorySnapshotStore(); + const version = 4; + const scope = createGraphKnowledgeAssetScope(UAL, version); + const payload: Quad[] = [ + { subject: 'urn:late:a', predicate: 'http://schema.org/status', object: '"late-match"', graph: '' }, + { subject: 'urn:late:b', predicate: 'http://schema.org/status', object: '"late-match"', graph: '' }, + ]; + const digest = workspacePublicQuadsDigest(payload); + await snapshotStore.putSnapshot({ digest, quads: payload }); + + const operationRows = (shareId: string, publicCount: number, quadsDigest: string): Quad[] => { + const subject = `urn:dkg:share:${CG}:${shareId}`; + return [ + { subject, predicate: `${DKG}contentScopeVersion`, object: `"2"^^<${XSD_INTEGER}>`, graph: WS_META }, + { subject, predicate: `${DKG}kaUal`, object: UAL, graph: WS_META }, + { subject, predicate: `${DKG}assertionVersion`, object: `"${version}"^^<${XSD_INTEGER}>`, graph: WS_META }, + { subject, predicate: `${DKG}shareOperationId`, object: `"${shareId}"`, graph: WS_META }, + { subject, predicate: `${DKG}publicQuadsDigest`, object: `"${quadsDigest}"`, graph: WS_META }, + { subject, predicate: `${DKG}publicQuadsCount`, object: `"${publicCount}"^^<${XSD_INTEGER}>`, graph: WS_META }, + { subject, predicate: `${DKG}publicSnapshotRef`, object: `"${quadsDigest}"`, graph: WS_META }, + ]; + }; + + // Sixteen decoys that sort BEFORE the real operation and are rejected only + // by their triple count — exactly the rows that used to consume the window. + for (let index = 0; index < 16; index += 1) { + await store.insert(operationRows( + `aaa-decoy-${String(index).padStart(2, '0')}`, + payload.length + 1, + `sha256:decoy-${index}`, + )); + } + await store.insert(operationRows('zzz-real-operation', payload.length, digest)); + + const handler = new FinalizationHandler(store, undefined as never, { + publicSnapshotStore: snapshotStore, + }); + const verified = await (handler as unknown as { + verifyImmutableGraphScopedSnapshot(input: unknown): Promise<{ status: string } | undefined>; + }).verifyImmutableGraphScopedSnapshot({ + contextGraphId: CG, + scope, + publicTripleCount: payload.length, + expectedPublicQuadsDigest: undefined, + expectedMerkleRoot: computeFlatKCRootV10(payload, []), + ctx, + }); + + expect(verified?.status).toBe('verified'); }); }); diff --git a/packages/agent/test/swm-snapshot-sync.test.ts b/packages/agent/test/swm-snapshot-sync.test.ts index 306af03a8e..97feb827da 100644 --- a/packages/agent/test/swm-snapshot-sync.test.ts +++ b/packages/agent/test/swm-snapshot-sync.test.ts @@ -97,6 +97,63 @@ describe('SWM snapshot catch-up sync', () => { ); }); + it('drops oversized graph-scoped replacement metadata without retrying the sync forever', async () => { + const nodeADataDir = await tempDataDir(); + const nodeBDataDir = await tempDataDir(); + const nodeA = await createAgent(nodeADataDir, 'SnapshotSyncOversizeA'); + const nodeB = await createAgent(nodeBDataDir, 'SnapshotSyncOversizeB'); + const sourceSnapshots = new FileWorkspacePublicSnapshotStore(join(nodeADataDir, 'swm-public-snapshots')); + + const write = await nodeA.publisher.writeToWorkspace(CONTEXT_GRAPH, [ + { subject: ENTITY, predicate: 'http://schema.org/name', object: '"Oversize metadata guard"', graph: '' }, + ], { publisherPeerId: 'peer-a' }); + const metaGraph = contextGraphWorkspaceMetaGraphUri(CONTEXT_GRAPH); + const operationSubject = `urn:dkg:share:${CONTEXT_GRAPH}:${write.shareOperationId}`; + const publisherPeerPredicate = 'http://dkg.io/ontology/publisherPeerId'; + await nodeA.store.deleteByPattern({ + graph: metaGraph, + subject: operationSubject, + predicate: publisherPeerPredicate, + }); + const oversizedPeerId = 'x'.repeat(61_000); + await nodeA.store.insert([{ + graph: metaGraph, + subject: operationSubject, + predicate: publisherPeerPredicate, + object: `"${oversizedPeerId}"`, + }]); + + installSharedMemorySyncMock(nodeB, nodeA, sourceSnapshots); + const detailedSync = () => (nodeB as unknown as { + syncSharedMemoryFromPeerDetailed(peerId: string, contextGraphIds: string[]): Promise<{ + failedPeers: number; + failedPhases: number; + completedPhases: number; + }>; + }).syncSharedMemoryFromPeerDetailed(REMOTE_PEER, [CONTEXT_GRAPH]); + + await expect(detailedSync()).resolves.toMatchObject({ + failedPeers: 0, + failedPhases: 0, + }); + // A second complete round pins the no-loop property: the poisoned row is + // seen again, but the already materialized graph/head remains usable and + // the phase still completes instead of re-fetching forever. + await expect(detailedSync()).resolves.toMatchObject({ + failedPeers: 0, + failedPhases: 0, + }); + + await expect(nodeB.store.query( + `ASK { GRAPH <${metaGraph}> { ?head ` + + `"${write.shareOperationId}" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: true }); + await expect(nodeB.store.query( + `ASK { GRAPH <${metaGraph}> { <${operationSubject}> ` + + `<${publisherPeerPredicate}> "${oversizedPeerId}" } }`, + )).resolves.toMatchObject({ type: 'boolean', value: false }); + }); + it('does not insert dangling publicSnapshotRef metadata when remote snapshots are unavailable', async () => { const nodeADataDir = await tempDataDir(); const nodeBDataDir = await tempDataDir(); diff --git a/packages/agent/test/swm-ttl-v2-cleanup.test.ts b/packages/agent/test/swm-ttl-v2-cleanup.test.ts index b9caa1bb42..54d450b004 100644 --- a/packages/agent/test/swm-ttl-v2-cleanup.test.ts +++ b/packages/agent/test/swm-ttl-v2-cleanup.test.ts @@ -10,7 +10,7 @@ * resolveKnowledgeAssetWorkspaceHead), * 3. the operation's public snapshot graph. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { makeTestKaNumberAllocator } from './_helpers/ka-allocator.js'; import { DKGAgent } from '../src/index.js'; import { createEVMAdapter, getSharedContext, createProvider, takeSnapshot, revertSnapshot, HARDHAT_KEYS } from '../../chain/test/evm-test-context.js'; @@ -236,4 +236,35 @@ describe('SWM TTL cleanup of graph-scoped V2 operations', () => { expect(await graphTripleCount(store, assertionGraph)).toBe(0); expect(await graphTripleCount(store, seeded.snapshotGraph)).toBe(0); }, 60_000); + + it('honors an explicit public owner/name context graph during finalized cleanup', async () => { + const cg = '0x1111111111111111111111111111111111111111/public-finalized-cleanup'; + const listSpy = vi.spyOn(node, 'listContextGraphs').mockResolvedValue([{ + id: cg, + uri: `did:dkg:context-graph:${cg}`, + name: 'public-finalized-cleanup', + isSystem: false, + subscribed: true, + synced: true, + }]); + const originalQuery = store.query.bind(store); + const discoveryQueries: string[] = []; + const querySpy = vi.spyOn(store, 'query').mockImplementation(async (query, options) => { + if (options?.source === 'agent.finalizedSwmCleanup.discover') { + discoveryQueries.push(query); + } + return originalQuery(query, options); + }); + + try { + await node.runFinalizedSwmCleanupSweep(); + } finally { + querySpy.mockRestore(); + listSpy.mockRestore(); + } + + expect(discoveryQueries.some((query) => query.includes( + `GRAPH <${contextGraphSharedMemoryMetaUri(cg)}>`, + ))).toBe(true); + }); }); diff --git a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts index 5f0aebf3e9..582cb37f0f 100644 --- a/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts +++ b/packages/agent/test/sync-responder-swm-meta-ceiling.test.ts @@ -639,7 +639,7 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { await store.close(); }, 120_000); - it('legacy cutoff-less sessions keep the unfiltered store-paged compatibility fallback', async () => { + it('legacy cutoff-less sessions keep TTL joins out of the store-paged compatibility fallback', async () => { const cgId = 'meta-ceiling-legacy'; const metaGraph = `did:dkg:context-graph:${cgId}/_shared_memory_meta`; const store = new OxigraphStore(); @@ -647,6 +647,22 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { for (const opId of ['x', 'y', 'z']) { await store.insert(workspaceOpQuads(cgId, opId, `urn:l:${opId}`, metaGraph, iso)); } + const markedUal = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/1'; + const unmarkedUal = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ac/1'; + const markedHead = `${markedUal}#dkg-swm-head`; + const markedOp = `urn:dkg:share:${cgId}:marked`; + const unmarkedHead = `${unmarkedUal}#dkg-swm-head`; + const unmarkedOp = `urn:dkg:share:${cgId}:unmarked`; + await store.insert([ + ...graphScopedHeadQuads(cgId, metaGraph, markedUal, 'marked', iso), + { + graph: metaGraph, + subject: markedOp, + predicate: `${DKG_NS}finalizedSwmCleanupRoot`, + object: `"0x${'ab'.repeat(32)}"`, + }, + ...graphScopedHeadQuads(cgId, metaGraph, unmarkedUal, 'unmarked', iso), + ]); let legacyPagedQueries = 0; const originalQuery = store.query.bind(store); @@ -658,9 +674,10 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { normalized.includes('ORDER BY ?g ?s ?p ?o') && /OFFSET \d+/.test(normalized) ) { - // The legacy paged query must never carry the TTL join. + // The legacy paged query must never carry the TTL join. Its only extra + // filters strip local GC metadata, not the active SWM lifecycle. expect(normalized).not.toContain('publishedAt'); - expect(normalized).not.toContain('FILTER'); + expect(normalized).toContain('finalizedSwmCleanupRoot'); legacyPagedQueries += 1; } return originalQuery(sparql, options as never); @@ -678,7 +695,13 @@ describe('SWM meta lane above the 64,000-row snapshot ceiling (#1847)', () => { { contextGraphId: cgId, includeSharedMemory: true, phase: 'meta', limit: 4, syncSessionId: 'legacy' }, 4, ); - expect(lines.size).toBe(15); + const joined = [...lines].join('\n'); + expect(lines.size).toBe(37); + expect(joined).toContain(markedHead); + expect(joined).toContain(markedOp); + expect(joined).not.toContain('finalizedSwmCleanupRoot'); + expect(joined).toContain(unmarkedHead); + expect(joined).toContain(unmarkedOp); expect(legacyPagedQueries).toBeGreaterThan(0); await store.close(); }); diff --git a/packages/agent/test/sync-responder-swm-subgraphs.test.ts b/packages/agent/test/sync-responder-swm-subgraphs.test.ts index 6fe23b9c7e..a19a6b77b7 100644 --- a/packages/agent/test/sync-responder-swm-subgraphs.test.ts +++ b/packages/agent/test/sync-responder-swm-subgraphs.test.ts @@ -408,6 +408,235 @@ describe('sync responder workspace branch — sub-graph SWM coverage', () => { expect(out).not.toContain('urn:dkg:share:devnet-test:stale-rootless'); await storeTtl.close(); }); + + it.each([0, 5_000])( + 'keeps pending SWM syncable but never advertises local GC metadata (ttl=%s)', + async (sharedMemoryTtlMs) => { + const markedStore = new OxigraphStore(); + const publishedAt = new Date(Date.now() - 1_000).toISOString(); + const ual = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ab/9'; + const opId = 'finalized-deferred-cleanup'; + const op = `urn:dkg:share:${CG_ID}:${opId}`; + const cleanupTask = 'urn:dkg:finalized-swm-cleanup:responder-test'; + const head = `${ual}#dkg-swm-head`; + const assertionGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ab/9`; + const markedAt = new Date(Date.now() - 2_000).toISOString(); + const markedEntity = 'urn:swm:finalized:still-syncable'; + const unmarkedGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ac/10`; + const unmarkedRoot = 'urn:swm:unmarked:must-sync'; + const unmarkedOp = `urn:dkg:share:${CG_ID}:unmarked-data`; + const tupleRows = (subject: string) => [ + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}contentScopeVersion`, object: '"2"^^' }, + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}kaUal`, object: ual }, + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}assertionVersion`, object: '"9"^^' }, + { graph: ROOT_SWM_META, subject, predicate: `${DKG_NS}shareOperationId`, object: `"${opId}"` }, + ]; + await markedStore.insert([ + { graph: ROOT_SWM_META, subject: op, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}rootEntity`, object: markedEntity }, + ...tupleRows(op), + ...tupleRows(head), + ...tupleRows(cleanupTask), + { graph: ROOT_SWM_META, subject: head, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: RDF_TYPE, object: `${DKG_NS}FinalizedSwmCleanupTask` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}assertionGraph`, object: assertionGraph }, + // The task subject carries all three local predicates, exactly as + // buildFinalizedSwmCleanupTaskQuads writes them. + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupMarkedAt`, object: `"${markedAt}"^^` }, + { graph: ROOT_SWM_META, subject: cleanupTask, predicate: `${DKG_NS}finalizedSwmCleanupHeadFingerprint`, object: `"${'cd'.repeat(32)}"` }, + // The OPERATION subject is the one that matters for the predicate + // guard. It is served to peers, so the subject-level + // `FILTER NOT EXISTS { ?s a FinalizedSwmCleanupTask }` does not cover + // it and only the predicate list keeps these rows local. + // + // Root and markedAt are what markFinalizedGraphScopedSwmForCleanup + // actually writes here (finalization-handler.ts:1602-1612). The head + // fingerprint is NOT written on an operation subject today — it is + // seeded anyway so the predicate guard is pinned as load-bearing + // rather than incidental: a future writer must not be able to leak it + // by putting it on a subject the responder serves. + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupRoot`, object: `"0x${'ab'.repeat(32)}"` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupMarkedAt`, object: `"${markedAt}"^^` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}finalizedSwmCleanupHeadFingerprint`, object: `"${'cd'.repeat(32)}"` }, + { graph: assertionGraph, subject: markedEntity, predicate: 'http://schema.org/name', object: '"finalized-copy"' }, + { graph: unmarkedGraph, subject: unmarkedRoot, predicate: 'http://schema.org/name', object: '"live-copy"' }, + { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, + { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + { graph: ROOT_SWM_META, subject: unmarkedOp, predicate: `${DKG_NS}rootEntity`, object: unmarkedRoot }, + ]); + const markedCap = captureHandler(); + registerSyncHandler({ + register: markedCap.register, + protocolSync: '/origintrail/dkg/sync/1.0.0', + syncDeniedResponse: 'sync-denied', + syncPageSize: 5000, + sharedMemoryTtlMs, + store: markedStore, + peerId: 'self-peer', + parseSyncRequest: (data) => JSON.parse(new TextDecoder().decode(data)) as SyncRequestEnvelope, + authorizeSyncRequest: async () => true, + logWarn: noopLog, + logDebug: noopLog, + }); + + const out = await markedCap.invoke({ + contextGraphId: CG_ID, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + + // Every local predicate, asserted by name. One predicate standing in + // for three is how two thirds of this guard went unheld: the fixtures + // only ever carried `finalizedSwmCleanupRoot`, so dropping either other + // entry from the filter leaked it to peers with the suite still green. + const expectNoLocalGcLeak = (payload: string) => { + expect(payload).not.toContain('finalizedSwmCleanupRoot'); + expect(payload).not.toContain('finalizedSwmCleanupMarkedAt'); + expect(payload).not.toContain('finalizedSwmCleanupHeadFingerprint'); + expect(payload).not.toContain(cleanupTask); + }; + + expect(out).toContain(head); + expect(out).toContain(op); + // Positive half: filtering everything would satisfy the absence checks. + expect(out).toContain(`${DKG_NS}shareOperationId`); + expect(out).toContain(`${DKG_NS}assertionGraph`); + expectNoLocalGcLeak(out); + + // The TTL-disabled lane serves two different readers: without a session + // it store-pages via readSwmMetaRowsPage, with one it loads the bounded + // snapshot. They filter in different places — store-side SPARQL vs the + // in-process strip — so a per-predicate hole can exist in one and not + // the other. + const sessionOut = await markedCap.invoke({ + contextGraphId: CG_ID, + syncSessionId: `finalized-cleanup-session-${sharedMemoryTtlMs}`, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + expect(sessionOut).toContain(op); + expectNoLocalGcLeak(sessionOut); + + const dataOut = await markedCap.invoke({ + contextGraphId: CG_ID, + syncSessionId: `finalized-cleanup-data-${sharedMemoryTtlMs}`, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'data', + }); + expect(dataOut).toContain(markedEntity); + expect(dataOut).toContain('"finalized-copy"'); + expect(dataOut).toContain(unmarkedRoot); + expect(dataOut).toContain('"live-copy"'); + + // After idle cleanup removes the active head/task, immutable operation + // history remains syncable with only its local tombstone stripped. + await markedStore.deleteByPattern({ + graph: ROOT_SWM_META, + subject: head, + }); + await markedStore.deleteByPattern({ + graph: ROOT_SWM_META, + subject: cleanupTask, + }); + const afterCleanup = await markedCap.invoke({ + contextGraphId: CG_ID, + syncSessionId: `finalized-cleanup-drained-${sharedMemoryTtlMs}`, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + expect(afterCleanup).toContain(op); + expectNoLocalGcLeak(afterCleanup); + await markedStore.close(); + }, + ); + + /** + * One assertion per predicate, so the three entries in the local-cleanup + * filter are pinned INDEPENDENTLY. + * + * The test above asserts all three inside one case, which proves none of + * them leaks but cannot show that the coverage discriminates: every + * single-predicate mutant kills exactly the same two cases, so an identical + * kill set is equally consistent with one assertion doing all the work. + * Splitting per predicate makes the kill sets differ — dropping + * `finalizedSwmCleanupMarkedAt` from the filter reddens only the markedAt + * rows, and likewise for the other two. + * + * Seeded on the OPERATION subject deliberately. The subject-level + * `FILTER NOT EXISTS { ?s a FinalizedSwmCleanupTask }` removes whole task + * subjects, so a predicate placed there is stripped whatever the predicate + * list says, and the assertion would hold with the entry deleted. + */ + it.each([ + ['finalizedSwmCleanupRoot', 0], + ['finalizedSwmCleanupMarkedAt', 0], + ['finalizedSwmCleanupHeadFingerprint', 0], + ['finalizedSwmCleanupRoot', 5_000], + ['finalizedSwmCleanupMarkedAt', 5_000], + ['finalizedSwmCleanupHeadFingerprint', 5_000], + ] as const)( + 'never advertises the %s local cleanup predicate on a served subject (ttl=%s)', + async (leakedPredicate, sharedMemoryTtlMs) => { + const store = new OxigraphStore(); + const publishedAt = new Date(Date.now() - 1_000).toISOString(); + const ual = 'did:dkg:hardhat:31337/0x00000000000000000000000000000000000000ad/11'; + const opId = 'per-predicate-leak'; + const op = `urn:dkg:share:${CG_ID}:${opId}`; + const assertionGraph = `${ROOT_SWM}/0x00000000000000000000000000000000000000ad/11`; + const rootEntity = 'urn:swm:per-predicate:root'; + await store.insert([ + { graph: ROOT_SWM_META, subject: op, predicate: RDF_TYPE, object: `${DKG_NS}WorkspaceOperation` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}publishedAt`, object: `"${publishedAt}"^^` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}rootEntity`, object: rootEntity }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}contentScopeVersion`, object: '"2"^^' }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}kaUal`, object: ual }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}assertionVersion`, object: '"11"^^' }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}shareOperationId`, object: `"${opId}"` }, + { graph: ROOT_SWM_META, subject: op, predicate: `${DKG_NS}${leakedPredicate}`, object: `"local-only-${leakedPredicate}"` }, + { graph: assertionGraph, subject: rootEntity, predicate: 'http://schema.org/name', object: '"served-payload"' }, + ]); + const cap = captureHandler(); + registerSyncHandler({ + register: cap.register, + protocolSync: '/origintrail/dkg/sync/1.0.0', + syncDeniedResponse: 'sync-denied', + syncPageSize: 5000, + sharedMemoryTtlMs, + store, + peerId: 'self-peer', + parseSyncRequest: (data) => JSON.parse(new TextDecoder().decode(data)) as SyncRequestEnvelope, + authorizeSyncRequest: async () => true, + logWarn: noopLog, + logDebug: noopLog, + }); + + const out = await cap.invoke({ + contextGraphId: CG_ID, + offset: 0, + limit: 5000, + includeSharedMemory: true, + phase: 'meta', + }); + + // Positive first: the subject IS served, so absence of the predicate + // cannot be explained by the whole subject being filtered out. + expect(out).toContain(op); + expect(out).toContain(`${DKG_NS}shareOperationId`); + expect(out).not.toContain(leakedPredicate); + expect(out).not.toContain(`local-only-${leakedPredicate}`); + await store.close(); + }, + ); }); // Codex review on #885 — URI shape alone is NOT a sufficient CG diff --git a/packages/agent/test/workspace-ttl.test.ts b/packages/agent/test/workspace-ttl.test.ts index 31263c7aba..517d48ff35 100644 --- a/packages/agent/test/workspace-ttl.test.ts +++ b/packages/agent/test/workspace-ttl.test.ts @@ -2,12 +2,13 @@ * Tests for workspace TTL / expiry: expired workspace operations are cleaned * up and not served to peers during sync. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { makeTestKaNumberAllocator } from "./_helpers/ka-allocator.js"; import { DKGAgent } from '../src/index.js'; import { createEVMAdapter, getSharedContext, createProvider, takeSnapshot, revertSnapshot, HARDHAT_KEYS } from '../../chain/test/evm-test-context.js'; import { mintTokens } from '../../chain/test/hardhat-harness.js'; import { ethers } from 'ethers'; +import type { TripleStore } from '@origintrail-official/dkg-storage'; let _fileSnapshot: string; beforeAll(async () => { @@ -89,14 +90,14 @@ describe('Workspace TTL', () => { }, 20000); }); -describe('setSharedMemoryTtlMs timer lifecycle', () => { +describe('setSharedMemoryTtlMs maintenance timer lifecycle', () => { let node: DKGAgent; afterAll(async () => { try { await node?.stop(); } catch {} }); - it('starts cleanup timer when TTL transitions from 0 to positive', async () => { + it('keeps finalized-SWM maintenance active when TTL expiry is disabled', async () => { node = await DKGAgent.create({ kaNumberAllocator: makeTestKaNumberAllocator(), name: 'TtlLifecycleNode', @@ -108,16 +109,53 @@ describe('setSharedMemoryTtlMs timer lifecycle', () => { await node.start(); await sleep(300); - // Timer should not be running (TTL=0) + // Finalized graph-scoped SWM GC has its own timer; ordinary TTL expiry is + // genuinely disabled. expect((node as any).swmCleanupTimer).toBeNull(); + expect((node as any).finalizedSwmCleanupTimer).not.toBeNull(); + + const store = (node as unknown as { store: TripleStore }).store; + let busy = true; + const pressureSpy = vi.spyOn(store, 'getPressureSnapshot').mockImplementation(() => ({ + ackInflight: 0, + healthInflight: 0, + normalInflight: busy ? 1 : 0, + backgroundInflight: 0, + ackQueued: 0, + healthQueued: 0, + normalQueued: 0, + backgroundQueued: 0, + maxConcurrent: 4, + ackReservedSlots: 1, + })); + const graphDiscoverySpy = vi.spyOn(node, 'listContextGraphs'); + + // A TTL-disabled periodic-style call exits before graph discovery while + // foreground work is active. + await expect(node.runFinalizedSwmCleanupSweep()).resolves.toMatchObject({ + pressureSkipped: true, + deletedItems: 0, + }); + expect(graphDiscoverySpy).not.toHaveBeenCalled(); + + // Once the store becomes idle, the same TTL-disabled maintenance path + // resumes graph discovery for deferred finalized-SWM cleanup. + busy = false; + await expect(node.runFinalizedSwmCleanupSweep()).resolves.toMatchObject({ + pressureSkipped: false, + }); + expect(graphDiscoverySpy).toHaveBeenCalledTimes(1); + pressureSpy.mockRestore(); + graphDiscoverySpy.mockRestore(); // Enable TTL at runtime node.setSharedMemoryTtlMs(60_000); expect((node as any).swmCleanupTimer).not.toBeNull(); - // Disable again + // Disabling TTL expiry must not disable finalized-SWM maintenance. node.setSharedMemoryTtlMs(0); expect((node as any).swmCleanupTimer).toBeNull(); + expect((node as any).finalizedSwmCleanupTimer).not.toBeNull(); }, 10000); }); diff --git a/packages/agent/vitest.unit.config.ts b/packages/agent/vitest.unit.config.ts index f2842594bb..d25c7309c0 100644 --- a/packages/agent/vitest.unit.config.ts +++ b/packages/agent/vitest.unit.config.ts @@ -12,6 +12,20 @@ const SQLITE_EXEC_ARGV = [ "--no-warnings=ExperimentalWarning", ]; +// These lifecycle suites use the shared chain fixture. Start Hardhat when the +// full unit inventory or one of those files is selected. Targeted pure-unit +// jobs (notably the RFC-64 Windows gate) keep their existing fast, chain-free +// path. +const explicitTestFilters = process.argv.filter((arg) => + /(?:^|[/\\])test[/\\]/.test(arg), +); +const needsAgentLifecycleFixture = explicitTestFilters.length === 0 + || explicitTestFilters.some((arg) => [ + "agent.part-16.test.ts", + "workspace-ttl.test.ts", + "swm-ttl-v2-cleanup.test.ts", + ].some((file) => arg.includes(file))); + export default defineConfig({ test: { include: [ @@ -101,6 +115,9 @@ export default defineConfig({ "test/finalization-reconcile-negative-memo.test.ts", "test/startup-jitter.test.ts", "test/finalization-lifecycle-logger.test.ts", + "test/finalized-swm-cleanup-worker.test.ts", + "test/finalized-swm-cleanup-sweep.test.ts", + "test/finalized-swm-cleanup-rotation.test.ts", "test/finalization-handler.test.ts", "test/finalization-handler-chain-truth.test.ts", "test/finalization-handler-defensive-cg-id.test.ts", @@ -109,6 +126,10 @@ export default defineConfig({ "test/finalization-recovery-sqlite-store.test.ts", "test/named-ka-publish-recovery.test.ts", "test/ka-graph-finalization-handler.test.ts", + "test/agent.part-16.test.ts", + "test/workspace-ttl.test.ts", + "test/swm-ttl-v2-cleanup.test.ts", + "test/sync-responder-swm-subgraphs.test.ts", "test/swm-slice-ka-bound.test.ts", "test/ka-lifecycle-asset-ual-timeout.test.ts", "test/storage-ack-lifecycle-identity.test.ts", @@ -139,6 +160,10 @@ export default defineConfig({ "test/replace-subject-agent-wrapper.test.ts", ], testTimeout: 60_000, + globalSetup: needsAgentLifecycleFixture + ? ["../chain/test/hardhat-global-setup.ts"] + : undefined, + env: needsAgentLifecycleFixture ? { HARDHAT_PORT: "9545" } : undefined, maxWorkers: 1, pool: "forks", execArgv: SQLITE_EXEC_ARGV, diff --git a/packages/cli/src/daemon/routes/agent-chat.ts b/packages/cli/src/daemon/routes/agent-chat.ts index 44cdeb0e0f..87ed84f4fe 100644 --- a/packages/cli/src/daemon/routes/agent-chat.ts +++ b/packages/cli/src/daemon/routes/agent-chat.ts @@ -1174,6 +1174,16 @@ export function buildSloPayload(agent: { deadlineExpired: number; pending: number; }; + getFinalizedSwmCleanupStats?: () => { + backlogDepth: number; + oldestMarkerAgeMs: number | null; + backlogStale: boolean; + pressureSkips: number; + deletedItems: number; + runs: number; + lastRunAt: string | null; + lastError: string | null; + }; }): { protocols: Record; gossip: { @@ -1220,11 +1230,28 @@ export function buildSloPayload(agent: { deadlineExpired: number; pending: number; }; + finalizedCleanup?: { + backlogDepth: number; + oldestMarkerAgeMs: number | null; + /** + * True while `backlogDepth`/`oldestMarkerAgeMs` are not a current + * whole-node measurement — the GC deferred on store pressure or its slice + * budget, or is part-way through a context-graph rotation. Depth 0 with + * this set means "unknown", not "drained". + */ + backlogStale: boolean; + pressureSkips: number; + deletedItems: number; + runs: number; + lastRunAt: string | null; + lastError: string | null; + }; }; } { const swmHandler = agent.getSwmHandlerStats(); const substrateFanout = agent.getSwmSubstrateFanoutStats?.(); const shareAckQuorum = agent.getSwmAckQuorumStats?.(); + const finalizedCleanup = agent.getFinalizedSwmCleanupStats?.(); return { protocols: agent.getMessengerSloStats(), gossip: agent.getSwmGossipStats(), @@ -1232,6 +1259,7 @@ export function buildSloPayload(agent: { ...swmHandler, ...(substrateFanout !== undefined ? { substrateFanout } : {}), ...(shareAckQuorum !== undefined ? { shareAckQuorum } : {}), + ...(finalizedCleanup !== undefined ? { finalizedCleanup } : {}), }, }; } diff --git a/packages/cli/test/api-slo-route.test.ts b/packages/cli/test/api-slo-route.test.ts index b01cbaa3ca..43400915d9 100644 --- a/packages/cli/test/api-slo-route.test.ts +++ b/packages/cli/test/api-slo-route.test.ts @@ -278,6 +278,49 @@ describe('/api/slo wire format (rc.9 PR-A / Codex PR #570 R10)', () => { expect((body as { swm: Record }).swm.shareAckQuorum).toBeUndefined(); }); + it('finalizedCleanup — exposes deferred GC backlog, pressure and progress', async () => { + const agent: FakeAgent = { + getMessengerSloStats: () => ({}), + getSwmGossipStats: () => ({ + publishFailures: {}, + publishFailuresOverflow: 0, + publishFailuresTruncated: false, + }), + getSwmHandlerStats: () => ({ + redundantApplies: {}, + redundantAppliesLowerBound: false, + redundantAppliesOverflow: 0, + redundantAppliesTruncated: false, + }), + getFinalizedSwmCleanupStats: () => ({ + backlogDepth: 12, + oldestMarkerAgeMs: 45_000, + backlogStale: true, + pressureSkips: 7, + deletedItems: 31, + runs: 9, + lastRunAt: '2026-07-31T10:00:00.000Z', + lastError: null, + }), + }; + ({ server, port } = await startSloServer(agent)); + + const { status, body } = await get(port, '/api/slo'); + expect(status).toBe(200); + expect((body as { swm: Record }).swm.finalizedCleanup).toEqual({ + backlogDepth: 12, + oldestMarkerAgeMs: 45_000, + // Deferred-vs-drained is the distinction operators page on; it must reach + // the wire, not just the in-process snapshot. + backlogStale: true, + pressureSkips: 7, + deletedItems: 31, + runs: 9, + lastRunAt: '2026-07-31T10:00:00.000Z', + lastError: null, + }); + }); + /** * rc.9 PR-D (codex follow-up from PR-G #G1): agents that ship * the `retryable` outcome bucket (transient receiver-side diff --git a/packages/publisher/src/index.ts b/packages/publisher/src/index.ts index 61fa9e63af..bf9f47455a 100644 --- a/packages/publisher/src/index.ts +++ b/packages/publisher/src/index.ts @@ -44,10 +44,13 @@ export { } from './catalog-trust.js'; export { resolveKnowledgeAssetWorkspaceHead, + sameKnowledgeAssetWorkspaceHead, resolveKnowledgeAssetOperationPublicQuads, resolveLiftWorkspaceSlice, storeKnowledgeAssetWorkspaceHead, storeKnowledgeAssetOperationPublicQuads, + workspaceKnowledgeAssetHeadSubject, + workspaceOperationSubject, KnowledgeAssetOperationPublicSnapshotNotFoundError, KnowledgeAssetWorkspaceHeadCorruptError, type KnowledgeAssetWorkspaceHead, diff --git a/packages/publisher/src/workspace-resolution.ts b/packages/publisher/src/workspace-resolution.ts index b2326ce404..694bc656e8 100644 --- a/packages/publisher/src/workspace-resolution.ts +++ b/packages/publisher/src/workspace-resolution.ts @@ -1,4 +1,4 @@ -import type { Quad, TripleStore } from '@origintrail-official/dkg-storage'; +import type { Quad, QueryOptions, TripleStore } from '@origintrail-official/dkg-storage'; import { GraphManager, PrivateContentStore } from '@origintrail-official/dkg-storage'; import { GRAPH_KA_CONTENT_SCOPE_VERSION, @@ -101,6 +101,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { contextGraphId: string; kaUal: string; subGraphName?: string; + queryOptions?: QueryOptions; }): Promise { const scope = createGraphKnowledgeAssetScope(params.kaUal, 1); const subGraphName = normalizeOptionalSubGraphName(params.subGraphName); @@ -128,6 +129,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { OPTIONAL { ?operation <${DKG}accessPolicy> ?accessPolicy } } } LIMIT 1`, + params.queryOptions, ); if (result.type !== 'bindings') { throw new Error( @@ -138,6 +140,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { const existence = await params.store.query( `ASK { GRAPH <${assertSafeIri(metaGraph)}> { ` + `<${assertSafeIri(subject)}> ?predicate ?object } }`, + params.queryOptions, ); if (existence.type !== 'boolean') { throw new Error( @@ -240,6 +243,7 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { const peersResult = await params.store.query( `SELECT ?peer WHERE { GRAPH <${assertSafeIri(metaGraph)}> { ` + `<${assertSafeIri(expectedOperationSubject)}> <${DKG}allowedPeer> ?peer } }`, + params.queryOptions, ); if (peersResult.type !== 'bindings') { throw new Error( @@ -272,6 +276,33 @@ export async function resolveKnowledgeAssetWorkspaceHead(params: { }; } +/** + * Canonical identity comparison for the current graph-scoped SWM head. + * + * Cleanup and writers share this helper so adding a lifecycle field cannot + * silently make a destructive cleanup compare less state than the workspace + * model owns. + */ +export function sameKnowledgeAssetWorkspaceHead( + left: KnowledgeAssetWorkspaceHead, + right: KnowledgeAssetWorkspaceHead, +): boolean { + const sameAllowedPeers = [...left.allowedPeers].sort().join('\0') + === [...right.allowedPeers].sort().join('\0'); + return left.kaUal === right.kaUal + && left.assertionVersion === right.assertionVersion + && left.assertionGraph === right.assertionGraph + && left.publicQuadsDigest === right.publicQuadsDigest + && left.publicTripleCount === right.publicTripleCount + && (left.privateMerkleRoot?.toLowerCase() ?? undefined) + === (right.privateMerkleRoot?.toLowerCase() ?? undefined) + && left.privateTripleCount === right.privateTripleCount + && left.shareOperationId === right.shareOperationId + && left.publisherPeerId === right.publisherPeerId + && left.accessPolicy === right.accessPolicy + && sameAllowedPeers; +} + /** Replace the durable current-assertion pointer after data and snapshot land. */ export async function storeKnowledgeAssetWorkspaceHead(params: { store: TripleStore; @@ -543,6 +574,12 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { assertionVersion: string | number | bigint; subGraphName?: string; publicSnapshotStore?: WorkspacePublicSnapshotStore; + /** + * Scheduler attribution for the reads below. Optional for callers that run + * off a request they already account for; supply it from lanes where this + * resolution is itself the cost being measured (finalization receipts). + */ + queryOptions?: QueryOptions; }): Promise { const expectedScope = createGraphKnowledgeAssetScope( params.kaUal, @@ -567,6 +604,7 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { OPTIONAL { <${assertSafeIri(subject)}> <${DKG}publisherPeerId> ?publisherPeerId } } } LIMIT 1`, + params.queryOptions, ); if (result.type !== 'bindings') { throw new Error( @@ -578,6 +616,7 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { const existence = await params.store.query( `ASK { GRAPH <${assertSafeIri(workspaceMetaGraph)}> { ` + `<${assertSafeIri(subject)}> ?predicate ?object } }`, + params.queryOptions, ); if (existence.type !== 'boolean') { throw new Error( @@ -632,7 +671,7 @@ export async function resolveKnowledgeAssetOperationPublicQuads(params: { `share operation ${params.shareOperationId}: unsafe snapshot graph`, ); } - quads = await resolveSnapshotGraphQuads(params.store, snapshotGraph); + quads = await resolveSnapshotGraphQuads(params.store, snapshotGraph, params.queryOptions); } if (!quads) { throw new KnowledgeAssetOperationPublicSnapshotNotFoundError( @@ -1124,7 +1163,7 @@ function normalizeOptionalSubGraphName(subGraphName: string | undefined): string return normalized; } -function workspaceOperationSubject(contextGraphId: string, shareOperationId: string): string { +export function workspaceOperationSubject(contextGraphId: string, shareOperationId: string): string { const normalizedContextGraphId = safeWorkspaceIdPart(contextGraphId, 'contextGraphId'); const normalizedShareOperationId = safeWorkspaceIdPart(shareOperationId, 'shareOperationId'); const subject = `urn:dkg:share:${normalizedContextGraphId}:${normalizedShareOperationId}`; @@ -1132,7 +1171,7 @@ function workspaceOperationSubject(contextGraphId: string, shareOperationId: str return subject; } -function workspaceKnowledgeAssetHeadSubject(kaUal: string): string { +export function workspaceKnowledgeAssetHeadSubject(kaUal: string): string { const scope = createGraphKnowledgeAssetScope(kaUal, 1); const subject = `${scope.ual}#dkg-swm-head`; assertSafeIri(subject); @@ -1177,9 +1216,14 @@ function workspaceKnowledgeAssetOperationSnapshotGraph( return graph; } -async function resolveSnapshotGraphQuads(store: TripleStore, snapshotGraph: string): Promise { +async function resolveSnapshotGraphQuads( + store: TripleStore, + snapshotGraph: string, + queryOptions?: QueryOptions, +): Promise { const result = await store.query( `CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <${assertSafeIri(snapshotGraph)}> { ?s ?p ?o } }`, + queryOptions, ); return result.type === 'quads' ? result.quads.map((quad) => ({ ...quad, graph: '' }))