diff --git a/packages/agent/src/chain-reconciler.ts b/packages/agent/src/chain-reconciler.ts index 51ed00901f..9a05118276 100644 --- a/packages/agent/src/chain-reconciler.ts +++ b/packages/agent/src/chain-reconciler.ts @@ -52,8 +52,11 @@ export type OrdinalOutcome = | { status: 'skip' }; export interface OrdinalRecoveryTarget { + localCgId: string; + onChainCgId: string; ordinal: number; ual: string; + merkleRoot: string; kaId: string; reason: 'no-swm' | 'verified-vm-metadata-pending'; } diff --git a/packages/agent/src/context-graph-binding-generation.ts b/packages/agent/src/context-graph-binding-generation.ts new file mode 100644 index 0000000000..771b3552b0 --- /dev/null +++ b/packages/agent/src/context-graph-binding-generation.ts @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 + +export function bumpContextGraphBindingGeneration( + generations: Map, + localCgId: string, +): number { + const generation = (generations.get(localCgId) ?? 0) + 1; + generations.set(localCgId, generation); + return generation; +} + +export function captureContextGraphBindingGeneration( + generations: Map, + localCgId: string, +): number { + return generations.get(localCgId) ?? 0; +} + +export function isContextGraphBindingGenerationCurrent( + generations: Map, + localCgId: string, + generation: number, +): boolean { + return captureContextGraphBindingGeneration(generations, localCgId) === generation; +} + +export function clearContextGraphBindingGeneration( + generations: Map, + localCgId: string, +): void { + generations.delete(localCgId); +} diff --git a/packages/agent/src/context-graph-membership-persist-scheduler.ts b/packages/agent/src/context-graph-membership-persist-scheduler.ts new file mode 100644 index 0000000000..7251342f06 --- /dev/null +++ b/packages/agent/src/context-graph-membership-persist-scheduler.ts @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: Apache-2.0 + +export class ContextGraphMembershipPersistQueueFullError extends Error { + readonly code = 'CG_MEMBERSHIP_PERSIST_QUEUE_FULL'; + + constructor(message: string) { + super(message); + this.name = 'ContextGraphMembershipPersistQueueFullError'; + } +} + +export class ContextGraphMembershipPersistQueueClosedError extends Error { + readonly code = 'CG_MEMBERSHIP_PERSIST_QUEUE_CLOSED'; + + constructor() { + super('Context-graph membership persistence is closed'); + this.name = 'ContextGraphMembershipPersistQueueClosedError'; + } +} + +export const CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_ERROR_CODE = + 'CG_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT'; + +export class ContextGraphMembershipPersistShutdownTimeoutError extends Error { + readonly code = CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_ERROR_CODE; + + constructor(timeoutMs: number) { + super(`Context-graph membership persistence did not drain within ${timeoutMs}ms`); + this.name = 'ContextGraphMembershipPersistShutdownTimeoutError'; + } +} + +interface PendingWrite { + strict: boolean; + write: () => Promise; + resolve: () => void; + reject: (error: unknown) => void; +} + +interface PersistLane { + active: boolean; + pending: PendingWrite[]; + drained: Promise; + resolveDrained: () => void; +} + +export interface ContextGraphMembershipPersistSchedulerStatus { + closed: boolean; + lanes: number; + active: number; + pending: number; +} + +/** + * Bounded keyed serialization for membership-store mutations. + * + * Strict operations preserve FIFO order and receive explicit backpressure. + * Adjacent background mutations coalesce to their latest write while the + * displaced caller settles successfully: those callers are deliberately + * best-effort, and only the final persisted state is meaningful. + */ +export class ContextGraphMembershipPersistScheduler { + private readonly lanes = new Map(); + private closed = false; + + constructor( + private readonly maxLanes = 1_000, + private readonly maxPendingPerLane = 16, + ) { + if (!Number.isSafeInteger(maxLanes) || maxLanes < 1) { + throw new Error('Membership persistence maxLanes must be a positive safe integer'); + } + if (!Number.isSafeInteger(maxPendingPerLane) || maxPendingPerLane < 1) { + throw new Error('Membership persistence maxPendingPerLane must be a positive safe integer'); + } + } + + enqueue( + key: string, + write: () => Promise, + options: { strict?: boolean } = {}, + ): Promise { + if (this.closed) { + return Promise.reject(new ContextGraphMembershipPersistQueueClosedError()); + } + + let lane = this.lanes.get(key); + if (!lane) { + if (this.lanes.size >= this.maxLanes) { + return Promise.reject(new ContextGraphMembershipPersistQueueFullError( + `Context-graph membership persistence reached its ${this.maxLanes}-lane limit`, + )); + } + let resolveDrained!: () => void; + const drained = new Promise((resolve) => { resolveDrained = resolve; }); + lane = { active: false, pending: [], drained, resolveDrained }; + this.lanes.set(key, lane); + } + + const strict = options.strict === true; + return new Promise((resolve, reject) => { + const tail = lane!.pending.at(-1); + if (!strict && tail && !tail.strict) { + tail.resolve(); + lane!.pending[lane!.pending.length - 1] = { strict, write, resolve, reject }; + } else { + if (lane!.pending.length >= this.maxPendingPerLane) { + reject(new ContextGraphMembershipPersistQueueFullError( + `Context-graph membership persistence key "${key}" reached its ` + + `${this.maxPendingPerLane}-write pending limit`, + )); + return; + } + lane!.pending.push({ strict, write, resolve, reject }); + } + if (!lane!.active) { + lane!.active = true; + void this.runLane(key, lane!); + } + }); + } + + closeAndDrain(): Promise { + this.closed = true; + return Promise.all([...this.lanes.values()].map((lane) => lane.drained)).then(() => undefined); + } + + reopen(): void { + if (this.lanes.size > 0) { + throw new Error('Cannot reopen context-graph membership persistence before it drains'); + } + this.closed = false; + } + + status(): ContextGraphMembershipPersistSchedulerStatus { + let active = 0; + let pending = 0; + for (const lane of this.lanes.values()) { + if (lane.active) active += 1; + pending += lane.pending.length; + } + return { closed: this.closed, lanes: this.lanes.size, active, pending }; + } + + private async runLane(key: string, lane: PersistLane): Promise { + while (lane.pending.length > 0) { + const operation = lane.pending.shift()!; + try { + await operation.write(); + operation.resolve(); + } catch (error) { + operation.reject(error); + } + } + lane.active = false; + if (this.lanes.get(key) === lane) this.lanes.delete(key); + lane.resolveDrained(); + } +} diff --git a/packages/agent/src/discovery.ts b/packages/agent/src/discovery.ts index ddeffd456a..ee37caad4c 100644 --- a/packages/agent/src/discovery.ts +++ b/packages/agent/src/discovery.ts @@ -59,11 +59,19 @@ export class DiscoveryClient { this.engine = engine; } - async findAgents(options: { framework?: string; limit?: number } = {}): Promise { + async findAgents(options: { + framework?: string; + agentAddress?: string; + limit?: number; + signal?: AbortSignal; + } = {}): Promise { let filter = ''; if (options.framework) { filter += `\n ?agent <${SKILL}framework> "${escapeSparqlLiteral(options.framework)}" .`; } + if (options.agentAddress) { + filter += `\n ?agent <${DKG}agentAddress> "${escapeSparqlLiteral(options.agentAddress)}" .`; + } const limitClause = options.limit ? `LIMIT ${options.limit}` : ''; @@ -80,7 +88,10 @@ export class DiscoveryClient { ${limitClause} `; - const result = await this.engine.query(sparql, { contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH }); + const result = await this.engine.query(sparql, { + contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH, + signal: options.signal, + }); return result.bindings.map((row) => ({ agentUri: row['agent'], @@ -93,6 +104,45 @@ export class DiscoveryClient { })); } + /** + * Deterministic, duplicate-free wallet-to-peer lookup for bounded recovery. + * Rich profile rows are deliberately not selected here: OPTIONAL profile + * properties can multiply rows before LIMIT and permanently hide a peer. + */ + async findAgentPeerIdsByAddress( + agentAddress: string, + options: { afterPeerId?: string; limit?: number; signal?: AbortSignal } = {}, + ): Promise { + const isEvmAddress = /^0x[0-9a-fA-F]{40}$/.test(agentAddress); + const addressMatch = isEvmAddress + ? `?agent <${DKG}agentAddress> ?storedAgentAddress . + FILTER(LCASE(STR(?storedAgentAddress)) = "${escapeSparqlLiteral(agentAddress.toLowerCase())}")` + : `?agent <${DKG}agentAddress> "${escapeSparqlLiteral(agentAddress)}" .`; + const limit = options.limit === undefined + ? undefined + : Math.max(1, Math.floor(options.limit)); + const afterFilter = options.afterPeerId + ? `FILTER(STR(?peerId) > "${escapeSparqlLiteral(options.afterPeerId)}")` + : ''; + const result = await this.engine.query(` + SELECT DISTINCT ?peerId WHERE { + ?agent a <${DKG}Agent> ; + <${DKG}peerId> ?peerId . + ${addressMatch} + ${afterFilter} + } + ORDER BY ASC(STR(?peerId)) + ${limit === undefined ? '' : `LIMIT ${limit}`} + `, { + contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH, + signal: options.signal, + }); + + return result.bindings + .map((row) => stripQuotes(row['peerId'] ?? '')) + .filter((peerId) => peerId.length > 0); + } + async findSkillOfferings(options: SkillSearchOptions = {}): Promise { const filters: string[] = []; @@ -144,7 +194,10 @@ export class DiscoveryClient { })); } - async findAgentByPeerId(peerId: string): Promise { + async findAgentByPeerId( + peerId: string, + options: { signal?: AbortSignal } = {}, + ): Promise { // Two-query path keeps the existing single-row SELECT semantics // for scalar columns (name, framework, nodeRole, relayAddress, // lastSeen) while a separate query gathers all `dkg:multiaddr` @@ -174,7 +227,10 @@ export class DiscoveryClient { LIMIT 1 `; - const scalarResult = await this.engine.query(scalar, { contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH }); + const scalarResult = await this.engine.query(scalar, { + contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH, + signal: options.signal, + }); if (scalarResult.bindings.length === 0) return null; const row = scalarResult.bindings[0]; @@ -202,7 +258,10 @@ export class DiscoveryClient { ${sparqlIri(safeAgentIri)} <${DKG}multiaddr> ?multiaddr . } `; - const multiResult = await this.engine.query(multiSparql, { contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH }); + const multiResult = await this.engine.query(multiSparql, { + contextGraphId: AGENT_REGISTRY_CONTEXT_GRAPH, + signal: options.signal, + }); const multiaddrs = multiResult.bindings .map((r) => (r['multiaddr'] ? stripQuotes(r['multiaddr']) : '')) .filter((s) => s.length > 0); diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index 16e7467c64..55bd1850a7 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -24,6 +24,7 @@ import type { Rfc64PublicCatalogServiceV1 } from './rfc64/public-catalog-service import type { Rfc64PublicCatalogNativeSynchronizationEvidenceV1 } from './rfc64/public-catalog-native-receiver-v1.js'; import { Rfc64PublicCatalogReconciliationFailureRegistryV1 } from './rfc64/public-catalog-reconciliation-failure-v1.js'; import { resolveVmReconcileStartupMaxDelayMs } from './startup-jitter.js'; +import { ContextGraphMembershipPersistScheduler } from './context-graph-membership-persist-scheduler.js'; import { DKGNode, ProtocolRouter, GossipSubManager, TypedEventBus, DKGEvent, LibP2PNetwork, PeerResolver, StubNetworkStateRegistry, @@ -362,6 +363,7 @@ import { type ContextGraphSubscriptionRehydrationStatus, type ContextGraphSubscriptionStore, type VmReconcileNegativeRecord, + type VmReconcileRotationRecord, type ContextGraphMemberPrincipalType, type ContextGraphMemberStatus, type ContextGraphMembershipRecord, @@ -419,6 +421,11 @@ function readNonNegativeNumberEnv(name: string, fallback: number): number { return Number.isFinite(parsed) ? Math.max(0, parsed) : fallback; } +function readPositiveSafeIntegerEnv(name: string, fallback: number): number { + const parsed = Number(process.env[name]); + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback; +} + export function createListContextGraphsCacheInvalidatingStore( innerStore: TripleStore, invalidate: () => void, @@ -898,14 +905,26 @@ export class DKGAgentBase { ); static readonly VM_RECONCILE_NEGATIVE_BACKOFF_BASE_MS = Math.max(5_000, DKGAgentBase.VM_RECONCILE_SWEEP_INTERVAL_MS); - static readonly VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS = - Number(process.env['DKG_VM_RECONCILE_BACKOFF_MAX_MS']) || 10 * 60_000; - static readonly VM_RECONCILE_CACHE_MAX_ENTRIES = - Math.max(1, Number(process.env['DKG_VM_RECONCILE_CACHE_MAX_ENTRIES']) || 1_000); + static readonly VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS = (() => { + const configured = Number(process.env['DKG_VM_RECONCILE_BACKOFF_MAX_MS']); + return Number.isFinite(configured) && configured > 0 + ? configured + : 10 * 60_000; + })(); + static readonly VM_RECONCILE_CACHE_MAX_ENTRIES = readPositiveSafeIntegerEnv( + 'DKG_VM_RECONCILE_CACHE_MAX_ENTRIES', + 1_000, + ); static readonly VM_RECONCILE_SWM_GEN_FINGERPRINT_MAX_ROWS = Math.max(1, Number(process.env['DKG_VM_RECONCILE_SWM_GEN_FINGERPRINT_MAX_ROWS']) || 2_000); - static readonly VM_RECONCILE_CG_STATE_MAX_ENTRIES = - Math.max(1, Number(process.env['DKG_VM_RECONCILE_CG_STATE_MAX_ENTRIES']) || 1_000); + static readonly VM_RECONCILE_CG_STATE_MAX_ENTRIES = readPositiveSafeIntegerEnv( + 'DKG_VM_RECONCILE_CG_STATE_MAX_ENTRIES', + 1_000, + ); + /** Maximum peers connected/probed/transported by one exact-recovery pass. */ + static readonly VM_RECONCILE_EXACT_PEER_MAX = 3; + /** Bounded proof universe retained across passes; transport still uses the cap above. */ + static readonly VM_RECONCILE_EXACT_ROSTER_MAX = MAX_CONTEXT_GRAPH_PARTICIPANT_AGENTS; static readonly VM_RECONCILE_QUEUE_MAX_PENDING = Math.max(1, Number(process.env['DKG_VM_RECONCILE_QUEUE_MAX_PENDING']) || 256); /** @@ -968,6 +987,17 @@ export class DKGAgentBase { protected vmReconcileTimer: ReturnType | null = null; /** Phase B — unified per-CG coalescing and node-wide admission policy. */ protected vmReconcileDispatcher?: VmReconcileDispatcher; + /** Closed dispatcher retained until every physically active worker settles. */ + protected vmReconcileRetirement: Promise | null = null; + /** Reconcile engines may outlive a caller's abort race; stop drains these before store teardown. */ + protected readonly vmReconcilePhysicalRuns = new Set>(); + /** Admitted authenticated graph-scoped stores must physically drain before backing-store teardown. */ + protected readonly graphScopedStorePhysicalRuns = new Set>(); + protected graphScopedStoreClosed = false; + /** True only after startup has installed every dependency the VM worker uses. */ + protected vmReconcileRuntimeReady = false; + /** A timed-out physical retirement quarantines this instance until stop is retried. */ + protected vmReconcileShutdownBlocked = false; /** Next eligible CG index for bounded periodic-sweep admission. */ protected vmReconcileSweepCursor = 0; /** Deterministically staggered cold-start prime, separate from the interval. */ @@ -990,9 +1020,27 @@ export class DKGAgentBase { protected coreHostRecordingGeneration = 0; /** Phase D/A4 — per-UAL retry damping after a chain ordinal has no matching local SWM snapshot. */ protected readonly vmReconcileNegativeCache = new Map>(); - /** Keys already consulted in the durable store during this process lifetime. */ - protected readonly vmReconcileNegativeCacheHydrated = new Set(); + /** Bounded access-ordered keys already consulted in the durable store. */ + protected readonly vmReconcileNegativeCacheHydrated = new Map(); protected readonly vmReconcileNegativeCacheKeysByCg = new Map>(); + /** Bounded, process-local clean-absence rotations for production VM recovery. */ + protected readonly vmReconcileRotationState = new Map(); + /** Next stable batch index to consider when the bounded rotation cache has waiters. */ + protected readonly vmReconcileRotationAdmissionCursorByCg = new Map(); + /** Last resolved curator peers, used to keep the capped exact-recovery roster authoritative. */ + protected readonly vmReconcileCuratorPeersByCg = new Map(); + /** Exclusive peer-id cursor used to walk oversized curator registries. */ + protected readonly vmReconcileCuratorPageCursorByCg = new Map(); + /** Bounded per-principal persistence lanes keep compensation ordered without heap backlog. */ + protected readonly contextGraphMembershipPersistence = new ContextGraphMembershipPersistScheduler(); + protected contextGraphMembershipPersistenceShutdownBlocked = false; + static readonly CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS = 5_000; + /** Late exact responses must not mutate rotation state after shutdown begins. */ + protected vmReconcileRotationClosed = false; + /** Monotonic guard: every VM reconcile continuation from an earlier node run stays stale. */ + protected vmReconcileLifecycleGeneration = 0; + /** Abortable boundary paired with the generation guard for restart-safe async work. */ + protected vmReconcileLifecycleController = new AbortController(); /** Phase D/A4 — per-CG active-fetch cooldown so one sweep cannot fan out repeated fetches. */ protected readonly vmReconcileFetchCooldownAt = new Map(); /** Phase D/A4 — round-robin cursor over the already ordered catch-up peer list. */ @@ -1057,6 +1105,8 @@ export class DKGAgentBase { /** Serialize local author-head construction/CAS independently per exact scope. */ protected readonly rfc64AuthorCatalogMutationQueuesV1 = new Map>(); protected readonly subscribedContextGraphs = new Map(); + /** Monotonic per-CG fence for async work captured across an on-chain binding transition. */ + protected readonly contextGraphBindingGenerations = new Map(); protected contextGraphSubscriptionRehydrationStatus: ContextGraphSubscriptionRehydrationStatus | null = null; protected readonly contextGraphSubscriptionRehydrationAccountedIds = new Set(); protected readonly contextGraphSubscriptionPersistRevisions = new Map(); diff --git a/packages/agent/src/dkg-agent-cg-resolve.ts b/packages/agent/src/dkg-agent-cg-resolve.ts index 6c514103b7..64b7322ace 100644 --- a/packages/agent/src/dkg-agent-cg-resolve.ts +++ b/packages/agent/src/dkg-agent-cg-resolve.ts @@ -653,7 +653,7 @@ export async function resolveCuratorSyncPeer( if (!resolved) { try { throwIfSyncAuthAborted(options.signal); - const agents = await agent.discovery.findAgents(); + const agents = await agent.discovery.findAgents({ signal: options.signal }); throwIfSyncAuthAborted(options.signal); const matches = agents.filter( (a) => a.agentAddress?.toLowerCase() === curatorIdentifier.toLowerCase(), @@ -789,12 +789,17 @@ export class ContextGraphResolveMethods extends DKGAgentBase { * ONTOLOGY/_meta count, and storage-backed graph presence also counts so local * shared-memory-only survivors are not treated as nonexistent. */ - async contextGraphExists(this: DKGAgent, contextGraphId: string): Promise { + async contextGraphExists( + this: DKGAgent, + contextGraphId: string, + options: { signal?: AbortSignal } = {}, + ): Promise { const contextGraphUri = `did:dkg:context-graph:${contextGraphId}`; const result = await this.store.query( `SELECT ?g WHERE { GRAPH ?g { <${contextGraphUri}> <${DKG_ONTOLOGY.RDF_TYPE}> <${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}> } } LIMIT 1`, + { signal: options.signal, source: 'agent.contextGraph.exists' }, ); if (result.type === 'bindings' && result.bindings.length > 0) { return true; @@ -822,7 +827,7 @@ export class ContextGraphResolveMethods extends DKGAgentBase { graphManager.sharedMemoryMetaUri(contextGraphId), ]; for (const graphUri of survivorGraphUris) { - if (await this.store.hasGraph(graphUri)) return true; + if (await this.store.hasGraph(graphUri, { signal: options.signal })) return true; } return false; } diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 01e2c88285..966fa1d961 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -105,6 +105,7 @@ import { runChangelogSync, planPageApply } from './sync/requester/changelog-sync import { authenticateVerifiedGraphScopedAsset, materializeVerifiedGraphScopedAsset, + type GraphScopedMaterializationOutcome, type VerifiedGraphScopedAsset, type VerifyContextGraphBinding, } from './sync/requester/graph-scoped-materialization.js'; @@ -248,7 +249,11 @@ import { waitForPeerProtocol } from './p2p/protocol-readiness.js'; import { orderCatchupPeers } from './p2p/peer-selection.js'; import { reconcileWarmCoreConnections, type WarmCoreAgent } from './p2p/warm-core-connections.js'; import { fetchSyncPages, type SyncPageResult } from './sync/requester/page-fetch.js'; -import { requireExactAssetUals } from './sync/exact-assets.js'; +import { + exactAssetFilterKey, + exactSyncPhaseAccumulationLimits, + requireExactAssetUals, +} from './sync/exact-assets.js'; import { insertWithOversizeGuard, type OversizeGuardHooks } from './sync/oversize-filter.js'; import { runOversizeSweep } from './sync/oversize-sweep.js'; import { getSyncCheckpointKey } from './sync/checkpoint/state.js'; @@ -260,8 +265,15 @@ import { } from './sync/requester/durable-sync-budget.js'; import { runDurableSync, + runDurableSyncDetailed, + type DetailedDurableSyncResult, + type DurableSyncContext, type VerifiedFullSnapshot, } from './sync/requester/durable-sync.js'; +import { + mergeExactDurableFetchDisposition, + type ExactDurableFetchDisposition, +} from './sync/requester/exact-durable-fetch.js'; import { resolveSyncAgentsMeta, shouldWithholdAgentsDurableMeta } from './sync/agents-meta-policy.js'; import { runSharedMemorySync, sharedMemoryOwnershipKeyFromGraph } from './sync/requester/shared-memory-sync.js'; import { createSharedMemorySnapshotMaterializer } from './sync/requester/swm-snapshot-materializer.js'; @@ -530,7 +542,14 @@ import { deserializePendingSenderKeyEntry, } from './dkg-agent-swm-state.js'; import { DKGAgentBase } from './dkg-agent-base.js'; +import { VmReconcileShutdownTimeoutError } from './vm-reconcile-service.js'; +import { ContextGraphMembershipPersistShutdownTimeoutError } from './context-graph-membership-persist-scheduler.js'; import type { DKGAgent } from './dkg-agent.js'; +import { + captureContextGraphBindingGeneration, + clearContextGraphBindingGeneration, + isContextGraphBindingGenerationCurrent, +} from './context-graph-binding-generation.js'; import { deterministicStartupJitterMs, scheduleAfterStartupJitter } from './startup-jitter.js'; const DEFAULT_HOST_MODE_RECONCILE_JITTER_RATIO = 0.15; @@ -685,7 +704,7 @@ function syncPageFetchCoalescingKey(params: { params.sinceBatchId ?? null, params.recovery === true, params.forceFreshSession === true, - params.assetUals ?? null, + params.assetUals === undefined ? null : exactAssetFilterKey(params.assetUals), ]); } @@ -789,6 +808,7 @@ function durableSyncSingleFlightKey(params: { hasAccessDeniedCallback: boolean; hasSinceBatchIdResolver: boolean; hasSignal: boolean; + hasCurrentFence: boolean; exactAssetUals?: readonly string[]; priority?: number; }): string | null { @@ -798,6 +818,7 @@ function durableSyncSingleFlightKey(params: { || params.hasAccessDeniedCallback || params.hasSinceBatchIdResolver || params.hasSignal + || params.hasCurrentFence ) { return null; } @@ -1016,6 +1037,8 @@ export type DurableSyncOptions = { * materialization check it before any subsequent commit boundary. */ signal?: AbortSignal; + /** Internal lifecycle fence for exact VM recovery. */ + isCurrent?: () => boolean; /** * Called synchronously after graph-scoped authentication succeeds and * immediately before atomic materialization is dispatched. This is a @@ -1041,6 +1064,16 @@ export type DurableSyncOptions = { source?: SyncAdmissionSource; }; +export interface ExactKnowledgeAssetSyncResult { + readonly result: DurableSyncResult; + readonly disposition: ExactDurableFetchDisposition; +} + +type PhysicalDurableSyncResult = { + readonly result: DurableSyncResult; + readonly exactFetchDisposition?: ExactDurableFetchDisposition; +}; + type LegacyDurableContextGraphOptions = { onPhase?: PhaseCallback; onAtomicCommitStarted?: (contextGraphId: string, ual: string) => void; @@ -1053,6 +1086,7 @@ type LegacyDurableContextGraphOptions = { authenticationTimeoutMs?: number; operationDeadline?: number; signal?: AbortSignal; + isCurrent?: () => boolean; }; const DURABLE_AUTHENTICATION_MAX_ATTEMPTS = 5; @@ -1409,7 +1443,18 @@ export class LifecycleSyncMethods extends DKGAgentBase { } async start(this: DKGAgent): Promise { + if (this.vmReconcileShutdownBlocked) { + throw new VmReconcileShutdownTimeoutError(DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS); + } + if (this.contextGraphMembershipPersistenceShutdownBlocked) { + throw new ContextGraphMembershipPersistShutdownTimeoutError( + DKGAgentBase.CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS, + ); + } if (this.started) return; + this.contextGraphMembershipPersistence.reopen(); + this.vmReconcileRuntimeReady = false; + this.graphScopedStoreClosed = false; this.coreHostRecordingGeneration += 1; this.coreHostRecordingsClosed = false; const ctx = createOperationContext('connect'); @@ -1453,6 +1498,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { ); } this.started = true; + this.openVmReconcileRotationState(); this.finalizationRuntime.markStarted({ localPeerId: this.peerId, localNodeIdentityId: this.identityId.toString(), @@ -3428,42 +3474,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (this.warmCoreTimer.unref) this.warmCoreTimer.unref(); } - // Phase B — chain-driven VM reconciliation. The coalescer collapses a burst - // of live KACG nudges for a CG into a single sweep; the periodic timer is - // the safety net that backfills missed events / transient fetch failures and - // catches up late subscribers (the "Monday Fun Facts" case). Only armed when - // the chain adapter exposes the per-CG registration-ordinal reads. - if (this.vmReconcileEnabled()) { - this.ensureVmReconcileDispatcher(); - const runSweep = (): void => { - this.runVmReconcileSweep().catch((err: unknown) => { - this.log.warn(ctx, `VM reconcile sweep failed: ${err instanceof Error ? err.message : String(err)}`); - }); - }; - // Prime once after a deterministic per-peer delay. A simultaneous - // four-node cold rollout must not turn into four synchronized Oxigraph - // scans; keeping this deterministic also makes restart behaviour and - // regression tests reproducible. - const startupDelayMs = deterministicStartupJitterMs( - `${this.node.peerId.toString()}\0${this.chain.chainId}`, - DKGAgentBase.VM_RECONCILE_STARTUP_MAX_DELAY_MS, - ); - this.vmReconcileStartupTimer = scheduleAfterStartupJitter( - () => { - this.vmReconcileStartupTimer = null; - runSweep(); - }, - startupDelayMs, - DKGAgentBase.VM_RECONCILE_SWEEP_INTERVAL_MS, - (timer) => { - this.vmReconcileTimer = timer; - if (timer.unref) timer.unref(); - }, - ); - if (this.vmReconcileStartupTimer.unref) this.vmReconcileStartupTimer.unref(); - this.log.info(ctx, `Chain-driven VM reconciliation armed (startupDelay ${startupDelayMs}ms, sweep ${DKGAgentBase.VM_RECONCILE_SWEEP_INTERVAL_MS}ms, depth ${DKGAgentBase.VM_RECONCILE_CONFIRMATION_DEPTH})`); - } - // rc.9 PR-10: dedicated join-approval retry tick removed. The // substrate's Messenger.processOutboxTick (set up immediately // below) now drives retries for /dkg/10.0.2/join-request the @@ -3517,6 +3527,38 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (rsStart === 'retryable') { this.scheduleRandomSamplingBindRetry(ctx); } + + // Arm VM work only at the final successful-start boundary. Every network, + // subscription, protocol, and persistence dependency is now initialized, + // and both initial start and same-object restart retain the cold-start + // jitter instead of launching an eager sweep against the old runtime. + this.vmReconcileRuntimeReady = true; + if (this.vmReconcileEnabled()) { + this.ensureVmReconcileDispatcher(); + const runSweep = (): void => { + this.runVmReconcileSweep().catch((err: unknown) => { + this.log.warn(ctx, `VM reconcile sweep failed: ${err instanceof Error ? err.message : String(err)}`); + }); + }; + const startupDelayMs = deterministicStartupJitterMs( + `${this.node.peerId.toString()}\0${this.chain.chainId}`, + DKGAgentBase.VM_RECONCILE_STARTUP_MAX_DELAY_MS, + ); + this.vmReconcileStartupTimer = scheduleAfterStartupJitter( + () => { + this.vmReconcileStartupTimer = null; + runSweep(); + }, + startupDelayMs, + DKGAgentBase.VM_RECONCILE_SWEEP_INTERVAL_MS, + (timer) => { + this.vmReconcileTimer = timer; + if (timer.unref) timer.unref(); + }, + ); + if (this.vmReconcileStartupTimer.unref) this.vmReconcileStartupTimer.unref(); + this.log.info(ctx, `Chain-driven VM reconciliation armed (startupDelay ${startupDelayMs}ms, sweep ${DKGAgentBase.VM_RECONCILE_SWEEP_INTERVAL_MS}ms, depth ${DKGAgentBase.VM_RECONCILE_CONFIRMATION_DEPTH})`); + } } /** @@ -4103,12 +4145,14 @@ export class LifecycleSyncMethods extends DKGAgentBase { peerId: string, ctx: OperationContext, label: string, + signal?: AbortSignal, ): Promise { if (this.networkAdmissionCoordinator.isAcceptedPeer(peerId)) return true; if (this.networkAdmissionCoordinator.isRejectedPeer(peerId)) return false; try { - return await this.networkAdmissionCoordinator.ensureAdmitted(peerId, ctx); + return await this.networkAdmissionCoordinator.ensureAdmitted(peerId, ctx, { signal }); } catch (err: unknown) { + if (signal?.aborted) throw err; const message = err instanceof Error ? err.message : String(err); this.log.warn(ctx, `${label} admission probe failed for ${peerId.slice(-8)}: ${message}`); return false; @@ -4634,93 +4678,144 @@ export class LifecycleSyncMethods extends DKGAgentBase { sinceBatchIdFor?: (contextGraphId: string) => string | undefined, options?: DurableSyncOptions, ): Promise { + return (await LifecycleSyncMethods.prototype.runLegacyDurableSyncDetailed.call( + this, + ctx, + remotePeerId, + contextGraphIds, + onPhase, + onAccessDenied, + sinceBatchIdFor, + options, + )).result; + } + + async runLegacyDurableSyncDetailed(this: DKGAgent, + ctx: OperationContext, + remotePeerId: string, + contextGraphIds: string[], + onPhase?: PhaseCallback, + onAccessDenied?: (contextGraphId: string) => void, + sinceBatchIdFor?: (contextGraphId: string) => string | undefined, + options?: DurableSyncOptions, + ): Promise { const syncAgentsMeta = resolveSyncAgentsMeta(this.config.syncAgentsMeta, process.env.DKG_SYNC_AGENTS_META); const stopOnBackoffWorthyFailure = options?.stopOnBackoffWorthyFailure; + const exactAssetUals = options?.exactAssetUals === undefined + ? undefined + : requireExactAssetUals(options.exactAssetUals); const operationBoundary = createDurableSyncOperationBoundary({ totalTimeoutMs: options?.totalTimeoutMs, signal: options?.signal, }); const authenticationTimeoutMs = normalizeDurableSyncTimeoutMs(options?.totalTimeoutMs); - const fetchTimeoutMs = options?.exactAssetUals && options.totalTimeoutMs === undefined + const fetchTimeoutMs = exactAssetUals && options?.totalTimeoutMs === undefined ? EXACT_RECOVERY_DURABLE_TRANSFER_TIMEOUT_MS : authenticationTimeoutMs; const orderedContextGraphIds = orderContextGraphIdsByPriority( contextGraphIds, this.config.syncContextGraphPriorities, ); - const runSync = async () => finalizeDurableSyncCompletion(await runOrderedContextGraphSyncs({ - work: orderedContextGraphIds.map((contextGraphId) => ({ - contextGraphId, - lane: 'durable' as const, - operationId: `durable:${contextGraphId}:${remotePeerId.slice(-8)}`, - run: async (remainingContextGraphs) => durableSyncAccumulatorFromResult( - await LifecycleSyncMethods.prototype.runLegacyDurableSyncForContextGraph.call( - this, - ctx, - remotePeerId, - contextGraphId, - remainingContextGraphs, - { - onPhase, - onAtomicCommitStarted: options?.onAtomicCommitStarted, - onAccessDenied, - sinceBatchIdFor, - stopOnBackoffWorthyFailure, - fetchTimeoutMs, - exactAssetUals: options?.exactAssetUals, - authenticationTimeoutMs, - operationDeadline: operationBoundary.deadline, - signal: operationBoundary.signal, - }, - ), - ), - })), - priorities: this.config.syncContextGraphPriorities, - emptyResult: createDurableSyncAccumulator, - runWithAdmission: async (item, work) => { - try { - return await runSerializedDurableContextGraphSync( - this, - remotePeerId, - item.contextGraphId, - () => this.runContextGraphSyncWithBackpressure( + let exactFetchDisposition: ExactDurableFetchDisposition | undefined; + const markExactFetchIncomplete = () => { + if (exactAssetUals === undefined) return; + exactFetchDisposition = mergeExactDurableFetchDisposition( + exactFetchDisposition, + 'incomplete', + ); + }; + const runSync = async (): Promise => { + const accumulator = await runOrderedContextGraphSyncs({ + work: orderedContextGraphIds.map((contextGraphId) => ({ + contextGraphId, + lane: 'durable' as const, + operationId: `durable:${contextGraphId}:${remotePeerId.slice(-8)}`, + run: async (remainingContextGraphs) => { + const detailed = await LifecycleSyncMethods.prototype.runLegacyDurableSyncForContextGraphDetailed.call( + this, ctx, - item.contextGraphId, - item.lane, - item.operationId, - work, + remotePeerId, + contextGraphId, + remainingContextGraphs, { - priorityOverride: options?.priority, - operationSignal: operationBoundary.signal, - source: options?.source, + onPhase, + onAtomicCommitStarted: options?.onAtomicCommitStarted, + onAccessDenied, + sinceBatchIdFor, + stopOnBackoffWorthyFailure, + fetchTimeoutMs, + exactAssetUals, + authenticationTimeoutMs, + operationDeadline: operationBoundary.deadline, + signal: operationBoundary.signal, + isCurrent: options?.isCurrent, }, - ), - operationBoundary.signal, - ); - } catch (error) { - if (!operationBoundary.signal?.aborted) throw error; - return markDurableTerminalBoundary(createDurableSyncAccumulator(), false); - } - }, - merge: mergeDurableSyncAccumulatorInto, - markDeferred: (summary) => { - recordDurableSyncDiagnostics(summary, { deferredBackpressure: 1 }); - return markDurableTerminalBoundary(summary, false); - }, - // Preserve already-merged progress, but record that cancellation left - // requested Context Graphs unvisited so the aggregate cannot finalize - // as complete. - markSkipped: (summary) => markDurableTerminalBoundary(summary, false), - shouldContinue: () => !operationBoundary.signal?.aborted, - shouldStop: (part) => Boolean( - stopOnBackoffWorthyFailure - && durableSyncAccumulatorHasBackoffWorthyFailure(part), - ), - onDeferred: (item, error) => this.log.info( - ctx, - `Deferring durable sync at CG ${item.contextGraphId} due to local backpressure: ${error.message}`, - ), - })); + ); + if (detailed.exactFetchDisposition !== undefined) { + exactFetchDisposition = mergeExactDurableFetchDisposition( + exactFetchDisposition, + detailed.exactFetchDisposition, + ); + } + return durableSyncAccumulatorFromResult(detailed.result); + }, + })), + priorities: this.config.syncContextGraphPriorities, + emptyResult: createDurableSyncAccumulator, + runWithAdmission: async (item, work) => { + try { + return await runSerializedDurableContextGraphSync( + this, + remotePeerId, + item.contextGraphId, + () => this.runContextGraphSyncWithBackpressure( + ctx, + item.contextGraphId, + item.lane, + item.operationId, + work, + { + priorityOverride: options?.priority, + operationSignal: operationBoundary.signal, + source: options?.source, + }, + ), + operationBoundary.signal, + ); + } catch (error) { + if (!operationBoundary.signal?.aborted) throw error; + markExactFetchIncomplete(); + return markDurableTerminalBoundary(createDurableSyncAccumulator(), false); + } + }, + merge: mergeDurableSyncAccumulatorInto, + markDeferred: (summary) => { + markExactFetchIncomplete(); + recordDurableSyncDiagnostics(summary, { deferredBackpressure: 1 }); + return markDurableTerminalBoundary(summary, false); + }, + // Preserve already-merged progress, but record that cancellation left + // requested Context Graphs unvisited so the aggregate cannot finalize + // as complete. + markSkipped: (summary) => { + markExactFetchIncomplete(); + return markDurableTerminalBoundary(summary, false); + }, + shouldContinue: () => !operationBoundary.signal?.aborted, + shouldStop: (part) => Boolean( + stopOnBackoffWorthyFailure + && durableSyncAccumulatorHasBackoffWorthyFailure(part), + ), + onDeferred: (item, error) => this.log.info( + ctx, + `Deferring durable sync at CG ${item.contextGraphId} due to local backpressure: ${error.message}`, + ), + }); + return { + result: finalizeDurableSyncCompletion(accumulator), + ...(exactFetchDisposition ? { exactFetchDisposition } : {}), + }; + }; const singleFlightKey = durableSyncSingleFlightKey({ remotePeerId, @@ -4734,7 +4829,8 @@ export class LifecycleSyncMethods extends DKGAgentBase { hasAccessDeniedCallback: Boolean(onAccessDenied), hasSinceBatchIdResolver: Boolean(sinceBatchIdFor), hasSignal: Boolean(operationBoundary.signal), - exactAssetUals: options?.exactAssetUals, + hasCurrentFence: Boolean(options?.isCurrent), + exactAssetUals, priority: options?.priority, }); const runWithinBoundary = async () => { @@ -4755,17 +4851,34 @@ export class LifecycleSyncMethods extends DKGAgentBase { /** * Foreground VM repair for one bounded set of locally-missing KAs. * Upgraded peers serve only these descriptors/payload graphs. Responses from - * older peers are accepted for rolling compatibility, but runDurableSync - * filters them back to this exact set before verification or storage. + * older peers are accepted for rolling compatibility only when their legacy + * full-CG prefix fits the exact accumulation bounds and covers the requested + * descriptors. Other legacy responses remain incomplete, fail closed, and + * rotate to another candidate instead of being verified or stored. */ async syncExactKnowledgeAssetsFromPeer(this: DKGAgent, remotePeerId: string, contextGraphId: string, requestedAssetUals: string[], + options: { signal?: AbortSignal; isCurrent?: () => boolean } = {}, ): Promise { + return (await this.syncExactKnowledgeAssetsFromPeerDetailed( + remotePeerId, + contextGraphId, + requestedAssetUals, + options, + )).result; + } + + async syncExactKnowledgeAssetsFromPeerDetailed(this: DKGAgent, + remotePeerId: string, + contextGraphId: string, + requestedAssetUals: string[], + options: { signal?: AbortSignal; isCurrent?: () => boolean } = {}, + ): Promise { const assetUals = requireExactAssetUals(requestedAssetUals); const ctx = createOperationContext('sync'); - return this.runLegacyDurableSync( + const detailed = await this.runLegacyDurableSyncDetailed( ctx, remotePeerId, [contextGraphId], @@ -4777,8 +4890,14 @@ export class LifecycleSyncMethods extends DKGAgentBase { stopOnBackoffWorthyFailure: true, priority: 1_000, source: 'vm-recovery', + signal: options.signal, + isCurrent: options.isCurrent, }, ); + return { + result: detailed.result, + disposition: detailed.exactFetchDisposition ?? 'incomplete', + }; } /** Execute one legacy durable Context Graph after its caller owns admission. */ @@ -4789,6 +4908,23 @@ export class LifecycleSyncMethods extends DKGAgentBase { remainingContextGraphs: number, options: LegacyDurableContextGraphOptions = {}, ): Promise { + return (await LifecycleSyncMethods.prototype.runLegacyDurableSyncForContextGraphDetailed.call( + this, + ctx, + remotePeerId, + contextGraphId, + remainingContextGraphs, + options, + )).result; + } + + async runLegacyDurableSyncForContextGraphDetailed(this: DKGAgent, + ctx: OperationContext, + remotePeerId: string, + contextGraphId: string, + remainingContextGraphs: number, + options: LegacyDurableContextGraphOptions = {}, + ): Promise { const { onPhase, onAtomicCommitStarted, @@ -4801,7 +4937,15 @@ export class LifecycleSyncMethods extends DKGAgentBase { authenticationTimeoutMs = fetchTimeoutMs, operationDeadline, signal, + isCurrent, } = options; + const assertLifecycleCurrent = () => { + if (isCurrent?.() === false) { + throw asSyncFetchAbortError(new Error( + `Exact VM recovery lifecycle for ${contextGraphId} is no longer current`, + )); + } + }; const syncAgentsMeta = resolveSyncAgentsMeta(this.config.syncAgentsMeta, process.env.DKG_SYNC_AGENTS_META); // The CG name commitment is immutable for an on-chain slot. Prove a // local/on-chain binding once per durable-sync invocation, then reuse the @@ -4849,7 +4993,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { contextGraphId, remainingContextGraphs, }); - return runDurableSync({ + const durableContext: DurableSyncContext = { ctx, remotePeerId, contextGraphIds: [contextGraphId], @@ -4897,60 +5041,137 @@ export class LifecycleSyncMethods extends DKGAgentBase { signal: operationSignal, }); }, - storeGraphScopedAsset: async ({ + storeGraphScopedAsset: ({ asset, authenticationDeadline, signal: operationSignal, }) => { - const authentication = await authenticateDurableGraphScopedAsset({ - chain: this.chain, - asset, - verifyContextGraphBinding, - deadline: authenticationDeadline, - signal: operationSignal, - onRetry: (error, attempt, maxAttempts) => { - this.log.warn( - ctx, - `Retrying graph-scoped durable authentication for ${asset.ual} ` - + `after transient chain verification failure (${attempt}/${maxAttempts}): ` - + `${error instanceof Error ? error.message : String(error)}`, - ); - }, - }); - if (operationSignal?.aborted) { - throw asSyncFetchAbortError(operationSignal.reason); + const shutdownError = () => asSyncFetchAbortError(new Error( + `Graph-scoped store for ${asset.ual} is closed for node shutdown`, + )); + if (this.graphScopedStoreClosed) { + return Promise.reject(shutdownError()); } - const verifiedOnChainId = authentication.onChainContextGraphId; const subscription = this.subscribedContextGraphs.get(asset.contextGraphId); - if (verifiedOnChainId && subscription && subscription.onChainId !== verifiedOnChainId) { - this.bindSubscriptionOnChainId( + let bindingGeneration = captureContextGraphBindingGeneration( + this.contextGraphBindingGenerations, + asset.contextGraphId, + ); + const bindingIsCurrent = () => ( + this.subscribedContextGraphs.get(asset.contextGraphId) === subscription + && isContextGraphBindingGenerationCurrent( + this.contextGraphBindingGenerations, asset.contextGraphId, - subscription, - verifiedOnChainId, - ); - this.persistContextGraphSubscriptionState(asset.contextGraphId); - } - if (operationSignal?.aborted) { - throw asSyncFetchAbortError(operationSignal.reason); - } - onAtomicCommitStarted?.(asset.contextGraphId, asset.ual); - const outcome = await materializeVerifiedGraphScopedAsset({ - store: this.store, - asset: authentication.asset, - options: { - priority: 'background', - source: 'agent.durableSync.graphScopedMaterialization', + bindingGeneration, + ) + ); + const physicalStore = Promise.resolve().then(async (): Promise => { + const authentication = await authenticateDurableGraphScopedAsset({ + chain: this.chain, + asset, + verifyContextGraphBinding, + deadline: authenticationDeadline, signal: operationSignal, - }, - oversizeHooks: { - recordDrops: (drops, seam) => this.oversizeTombstoneLog.record(drops, seam), - }, + onRetry: (error, attempt, maxAttempts) => { + this.log.warn( + ctx, + `Retrying graph-scoped durable authentication for ${asset.ual} ` + + `after transient chain verification failure (${attempt}/${maxAttempts}): ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + }, + }); + if (operationSignal?.aborted) { + throw asSyncFetchAbortError(operationSignal.reason); + } + assertLifecycleCurrent(); + if (this.graphScopedStoreClosed) throw shutdownError(); + if (!bindingIsCurrent()) { + throw asSyncFetchAbortError(new Error( + `Context graph binding for ${asset.contextGraphId} changed during authentication`, + )); + } + const verifiedOnChainId = authentication.onChainContextGraphId; + assertLifecycleCurrent(); + if ( + verifiedOnChainId + && subscription?.onChainId + && subscription.onChainId !== verifiedOnChainId + ) { + throw Object.assign( + new Error( + `Graph-scoped durable sync ${asset.ual} belongs to on-chain context graph ` + + `${verifiedOnChainId}, but local ${asset.contextGraphId} is already bound to ` + + `${subscription.onChainId}`, + ), + { code: 'VM_CHAIN_CONTEXT_GRAPH_MISMATCH' }, + ); + } + if (verifiedOnChainId && subscription && subscription.onChainId === undefined) { + await this.persistContextGraphSubscriptionStrict( + asset.contextGraphId, + { ...subscription, onChainId: verifiedOnChainId, lastReconciledOrdinal: 0 }, + undefined, + bindingIsCurrent, + ); + assertLifecycleCurrent(); + if (!bindingIsCurrent()) { + throw asSyncFetchAbortError(new Error( + `Context graph binding for ${asset.contextGraphId} changed while its authenticated update was persisted`, + )); + } + this.bindSubscriptionOnChainId( + asset.contextGraphId, + subscription, + verifiedOnChainId, + ); + bindingGeneration = captureContextGraphBindingGeneration( + this.contextGraphBindingGenerations, + asset.contextGraphId, + ); + assertLifecycleCurrent(); + } + if (operationSignal?.aborted) { + throw asSyncFetchAbortError(operationSignal.reason); + } + assertLifecycleCurrent(); + if (!bindingIsCurrent()) { + throw asSyncFetchAbortError(new Error( + `Context graph binding for ${asset.contextGraphId} changed before materialization`, + )); + } + onAtomicCommitStarted?.(asset.contextGraphId, asset.ual); + if (this.graphScopedStoreClosed) throw shutdownError(); + const outcome = await materializeVerifiedGraphScopedAsset({ + store: this.store, + asset: authentication.asset, + isCurrent: () => (isCurrent?.() ?? true) && bindingIsCurrent(), + shouldQuarantineCommitted: () => { + const current = this.subscribedContextGraphs.get(asset.contextGraphId); + return (subscription !== undefined && current === undefined) + || (verifiedOnChainId !== null + && current !== undefined + && current.onChainId !== verifiedOnChainId); + }, + options: { + priority: 'background', + source: 'agent.durableSync.graphScopedMaterialization', + signal: operationSignal, + }, + oversizeHooks: { + recordDrops: (drops, seam) => this.oversizeTombstoneLog.record(drops, seam), + }, + }); + if (outcome === 'applied') { + this.invalidateListContextGraphsCache(); + this.contextGraphMetaProjection.markDirtyFromQuads(authentication.asset.metadataQuads); + } + return outcome; + }); + this.graphScopedStorePhysicalRuns.add(physicalStore); + return physicalStore.finally(() => { + this.graphScopedStorePhysicalRuns.delete(physicalStore); }); - if (outcome === 'applied') { - this.invalidateListContextGraphsCache(); - this.contextGraphMetaProjection.markDirtyFromQuads(authentication.asset.metadataQuads); - } - return outcome; }, onVerifiedFullSnapshot, deleteCheckpoint: (key) => this.syncCheckpoints.delete(key), @@ -4958,7 +5179,11 @@ export class LifecycleSyncMethods extends DKGAgentBase { logInfo: (opCtx, message) => this.log.info(opCtx, message), logWarn: (opCtx, message) => this.log.warn(opCtx, message), logDebug: (opCtx, message) => this.log.debug(opCtx, message), - }); + }; + if (exactAssetUals !== undefined) { + return runDurableSyncDetailed(durableContext); + } + return { result: await runDurableSync(durableContext) }; } /** @@ -5344,6 +5569,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { // responder-session identities so offsets can never cross asset batches. assetUals?: string[], ): Promise { + const exactAccumulationLimits = assetUals === undefined + ? undefined + : exactSyncPhaseAccumulationLimits(assetUals); // A caller signal defines an operation-owned cancellation contract. Do not // place those fetches in the shared page map: even equal wall-clock // deadlines do not make independently abortable operations compatible. @@ -5407,6 +5635,8 @@ export class LifecycleSyncMethods extends DKGAgentBase { snapshotRef, sinceBatchId, assetUals, + maxAcceptedBytes: exactAccumulationLimits?.maxBytes, + maxAcceptedQuads: exactAccumulationLimits?.maxQuads, deadline, recovery, syncPageTimeoutMs: SYNC_PAGE_TIMEOUT_MS, @@ -5505,39 +5735,145 @@ export class LifecycleSyncMethods extends DKGAgentBase { */ async resolveCuratorPeerIdsForCg(this: DKGAgent, contextGraphId: string, - ): Promise<{ peerIds: string[]; curatorIsLocal: boolean; legacyTripleResolved: boolean }> { + options: { + maxPeerIds?: number; + pagePeerIds?: number; + afterPeerId?: string; + signal?: AbortSignal; + isCurrent?: () => boolean; + } = {}, + ): Promise<{ + peerIds: string[]; + curatorIsLocal: boolean; + legacyTripleResolved: boolean; + lookupFailed?: boolean; + overflowed?: boolean; + nextPageAfterPeerId?: string; + }> { + const assertCurrent = (): void => { + if (options.signal?.aborted || options.isCurrent?.() === false) { + throw new DOMException('Curator discovery is no longer current', 'AbortError'); + } + }; + assertCurrent(); const structuralCuratorDid = deriveCuratorDidFromCgId(contextGraphId); if (structuralCuratorDid) { const structuralAgent = structuralCuratorDid.slice('did:dkg:agent:'.length).toLowerCase(); if ([...this.localAgents.keys()].some((addr) => addr.toLowerCase() === structuralAgent)) { return { peerIds: [], curatorIsLocal: true, legacyTripleResolved: false }; } - const resolve = async (): Promise => { - let agents: Array<{ agentAddress?: string; peerId: string }>; + const resolve = async (): Promise<{ + peerIds: string[]; + lookupFailed: boolean; + overflowed?: boolean; + nextPageAfterPeerId?: string; + }> => { + assertCurrent(); try { - agents = await this.discovery.findAgents(); + if (options.maxPeerIds !== undefined + && typeof this.discovery.findAgentPeerIdsByAddress === 'function') { + const pagePeerIds = Math.min( + options.maxPeerIds, + Math.max(1, Math.floor(options.pagePeerIds ?? options.maxPeerIds)), + ); + const queryPage = (afterPeerId?: string) => + this.discovery.findAgentPeerIdsByAddress(structuralAgent, { + ...(afterPeerId ? { afterPeerId } : {}), + limit: (afterPeerId ? pagePeerIds : options.maxPeerIds!) + 1, + signal: options.signal, + }); + let pageStartedAtBeginning = !options.afterPeerId; + let peerIds = await queryPage(options.afterPeerId); + assertCurrent(); + if (options.afterPeerId && peerIds.length === 0) { + pageStartedAtBeginning = true; + peerIds = await queryPage(); + } + assertCurrent(); + const overflowed = !pageStartedAtBeginning + || peerIds.length > options.maxPeerIds; + const bounded = peerIds.slice(0, overflowed ? pagePeerIds : options.maxPeerIds); + return { + peerIds: bounded, + lookupFailed: false, + overflowed, + ...(overflowed && bounded[0] + ? { nextPageAfterPeerId: bounded[bounded.length - 1] } + : {}), + }; + } + + const agents = await this.discovery.findAgents({ + agentAddress: structuralAgent, + signal: options.signal, + }); + assertCurrent(); + const peerIds = [...new Set(agents + .filter((a) => a.agentAddress?.toLowerCase() === structuralAgent) + .map((a) => a.peerId))] + .sort((left, right) => left.localeCompare(right)); + return { peerIds, lookupFailed: false, overflowed: false }; } catch { - agents = []; + assertCurrent(); + return { peerIds: [], lookupFailed: true }; } - return agents - .filter((a) => a.agentAddress?.toLowerCase() === structuralAgent) - .map((a) => a.peerId); }; - let curatorPeers = await resolve(); - if (curatorPeers.length === 0) { - await this.refreshMetaFromCurator(contextGraphId).catch(() => undefined); - curatorPeers = await resolve(); + let resolution = await resolve(); + assertCurrent(); + if (resolution.peerIds.length === 0) { + assertCurrent(); + const refreshed = await this.refreshMetaFromCurator(contextGraphId, { + signal: options.signal, + }).catch((error) => { + assertCurrent(); + return false; + }); + assertCurrent(); + resolution = await resolve(); + assertCurrent(); + // An empty local query after a failed/cooldown refresh is not an + // authoritative empty registry. Preserve any caller-side last-known + // curator roster unless another writer populated the registry. + if (!refreshed && !resolution.lookupFailed && resolution.peerIds.length === 0) { + resolution = { ...resolution, lookupFailed: true }; + } } - return { peerIds: curatorPeers, curatorIsLocal: false, legacyTripleResolved: false }; + return { + peerIds: resolution.peerIds, + curatorIsLocal: false, + legacyTripleResolved: false, + ...(resolution.lookupFailed ? { lookupFailed: true } : {}), + ...(resolution.overflowed ? { overflowed: true } : {}), + ...(resolution.nextPageAfterPeerId + ? { nextPageAfterPeerId: resolution.nextPageAfterPeerId } + : {}), + }; } // Legacy non-wallet-scoped CG: fall back to triple-based curator resolution. - if (await this.isCuratorOf(contextGraphId)) { + assertCurrent(); + if (await this.isCuratorOf(contextGraphId, { signal: options.signal })) { + assertCurrent(); return { peerIds: [], curatorIsLocal: true, legacyTripleResolved: true }; } - let curatorPeerId = await this.resolveCuratorPeerId(contextGraphId); + assertCurrent(); + let curatorPeerId = await this.resolveCuratorPeerId(contextGraphId, { + signal: options.signal, + }); + assertCurrent(); if (!curatorPeerId) { - await this.refreshMetaFromCurator(contextGraphId).catch(() => undefined); - curatorPeerId = await this.resolveCuratorPeerId(contextGraphId); + assertCurrent(); + await this.refreshMetaFromCurator(contextGraphId, { + signal: options.signal, + }) + .catch((error) => { + assertCurrent(); + return undefined; + }); + assertCurrent(); + curatorPeerId = await this.resolveCuratorPeerId(contextGraphId, { + signal: options.signal, + }); + assertCurrent(); } if (!curatorPeerId) return { peerIds: [], curatorIsLocal: false, legacyTripleResolved: true }; if (curatorPeerId === this.peerId) return { peerIds: [], curatorIsLocal: true, legacyTripleResolved: true }; @@ -6662,19 +6998,32 @@ export class LifecycleSyncMethods extends DKGAgentBase { ); } - async ensurePeerConnected(this: DKGAgent, peerId: string): Promise { - await ensurePeerConnectedAtom(this.node.libp2p as any, this.discovery, peerId); - if (await this.networkAdmissionCoordinator.ensureAdmitted(peerId, createOperationContext('connect'))) return; + async ensurePeerConnected( + this: DKGAgent, + peerId: string, + options: { signal?: AbortSignal } = {}, + ): Promise { + await ensurePeerConnectedAtom(this.node.libp2p as any, this.discovery, peerId, options); + if (await this.networkAdmissionCoordinator.ensureAdmitted( + peerId, + createOperationContext('connect'), + options, + )) return; throw new NetworkAdmissionRejectedError(peerId); } - async waitForSyncProtocol(this: DKGAgent, pid: { toString(): string }): Promise { + async waitForSyncProtocol( + this: DKGAgent, + pid: { toString(): string }, + signal?: AbortSignal, + ): Promise { return waitForPeerProtocol( this.node.libp2p.peerStore as any, pid, PROTOCOL_SYNC, SYNC_PROTOCOL_CHECK_ATTEMPTS, SYNC_PROTOCOL_CHECK_DELAY_MS, + signal, ); } @@ -6882,6 +7231,15 @@ export class LifecycleSyncMethods extends DKGAgentBase { return run; } + enqueueContextGraphMembershipPersistWrite( + this: DKGAgent, + key: string, + write: () => Promise, + options?: { strict?: boolean }, + ): Promise { + return this.contextGraphMembershipPersistence.enqueue(key, write, options); + } + finishContextGraphSubscriptionPersistRevision(this: DKGAgent, contextGraphId: string, revision?: number): void { if (revision == null) return; const pending = this.contextGraphSubscriptionPersistPendingRevisions.get(contextGraphId); @@ -7012,15 +7370,20 @@ export class LifecycleSyncMethods extends DKGAgentBase { deleteContextGraphSubscription(this: DKGAgent, contextGraphId: string): boolean { this.invalidateListContextGraphsCache(); this.forceClearVmReconcileStateForContextGraph(contextGraphId); - return this.subscribedContextGraphs.delete(contextGraphId); + const deleted = this.subscribedContextGraphs.delete(contextGraphId); + // Every in-flight binding continuation also captures the subscription + // object, so deleting this numeric generation cannot revive old work if a + // new subscription later reuses the same local id. + clearContextGraphBindingGeneration(this.contextGraphBindingGenerations, contextGraphId); + return deleted; } - persistContextGraphSubscriptionState(this: DKGAgent, contextGraphId: string): void { + persistContextGraphSubscriptionState(this: DKGAgent, contextGraphId: string): Promise { if (!this.config.contextGraphSubscriptionStore) { this.clearContextGraphSubscriptionPersistRevisionStateIfIdle(contextGraphId); - return; + return Promise.resolve(); } - this.persistContextGraphSubscription(contextGraphId, { + return this.persistContextGraphSubscription(contextGraphId, { revision: this.nextContextGraphSubscriptionPersistRevision(contextGraphId), updateRehydrationStatus: false, }); @@ -7032,12 +7395,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { revision?: number; updateRehydrationStatus?: boolean; }, - ): void { + ): Promise { this.invalidateListContextGraphsCache(); const store = this.config.contextGraphSubscriptionStore; if (!store) { this.clearContextGraphSubscriptionPersistRevisionStateIfIdle(contextGraphId); - return; + return Promise.resolve(); } const sub = this.subscribedContextGraphs.get(contextGraphId); // Persist member subscriptions AND (Phase D) public CGs this Core hosts — @@ -7046,7 +7409,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // row only when the node neither subscribes to nor hosts the CG. this.beginContextGraphSubscriptionPersistRevision(contextGraphId, options?.revision); if (!sub?.subscribed && !sub?.coreHosted) { - void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.delete(contextGraphId)) + return this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.delete(contextGraphId)) .then(() => { if ( options?.updateRehydrationStatus === true && @@ -7064,7 +7427,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { .finally(() => { this.finishContextGraphSubscriptionPersistRevision(contextGraphId, options?.revision); }); - return; } const record = { id: contextGraphId, @@ -7079,7 +7441,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { coreHosted: sub.coreHosted, syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), }; - void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.save(record)) + return this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.save(record)) .then(() => { if ( options?.updateRehydrationStatus === true && @@ -7101,6 +7463,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { contextGraphId: string, subscription?: ContextGraphSub, syncScoped?: boolean, + isCurrent: () => boolean = () => true, ): Promise { const store = this.config.contextGraphSubscriptionStore; if (!store) { @@ -7109,10 +7472,15 @@ export class LifecycleSyncMethods extends DKGAgentBase { // retains the backward-compatible in-memory approval path. return; } - const sub = subscription ?? this.subscribedContextGraphs.get(contextGraphId); - if (!sub?.subscribed) { + const expectedLiveSub = this.subscribedContextGraphs.get(contextGraphId); + const expectedBindingGeneration = captureContextGraphBindingGeneration( + this.contextGraphBindingGenerations, + contextGraphId, + ); + const sub = subscription ?? expectedLiveSub; + if (!sub?.subscribed && !sub?.coreHosted) { throw new Error( - `Cannot acknowledge join approval for "${contextGraphId}": active subscription state is missing`, + `Cannot persist context graph "${contextGraphId}": active subscription or host state is missing`, ); } const record = { @@ -7130,10 +7498,25 @@ export class LifecycleSyncMethods extends DKGAgentBase { }; // Queue behind any fire-and-forget writes scheduled by subscribe/mark so // this final authoritative snapshot is the last write before the ACK. - await this.enqueueContextGraphSubscriptionPersistWrite( - contextGraphId, - () => store.save(record), - ); + await this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, async () => { + const current = this.subscribedContextGraphs.get(contextGraphId); + if ( + !isCurrent() + || + current !== expectedLiveSub + || (!current?.subscribed && !current?.coreHosted) + || !isContextGraphBindingGenerationCurrent( + this.contextGraphBindingGenerations, + contextGraphId, + expectedBindingGeneration, + ) + ) { + throw asSyncFetchAbortError(new Error( + `Context graph "${contextGraphId}" changed before its strict subscription snapshot was persisted`, + )); + } + await store.save(record); + }); } /** @@ -7156,93 +7539,101 @@ export class LifecycleSyncMethods extends DKGAgentBase { membership.principalType, membership.principalId, ); + const membershipKey = `${contextGraphId}\0${membership.principalType}\0${normalizedPrincipalId}`; + await this.enqueueContextGraphMembershipPersistWrite(membershipKey, () => + this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, async () => { + let previousMembership: (ContextGraphMembershipRecord & { + firstSeenAt?: number; + updatedAt: number; + }) | null = null; + let previousMembershipKnown = membershipStore === undefined; + if (membershipStore?.loadAll) { + const rows = await membershipStore.loadAll(); + previousMembershipKnown = true; + previousMembership = rows.find((row) => + row.contextGraphId === contextGraphId + && row.principalType === membership.principalType + && this.normalizeMembershipPrincipal(row.principalType, row.principalId) === normalizedPrincipalId + ) ?? null; + } - let previousMembership: (ContextGraphMembershipRecord & { - firstSeenAt?: number; - updatedAt: number; - }) | null = null; - let previousMembershipKnown = membershipStore === undefined; - if (membershipStore?.loadAll) { - const rows = await membershipStore.loadAll(); - previousMembershipKnown = true; - previousMembership = rows.find((row) => - row.contextGraphId === contextGraphId && - row.principalType === membership.principalType && - this.normalizeMembershipPrincipal(row.principalType, row.principalId) === normalizedPrincipalId - ) ?? null; - } - - let previousSubscription: ContextGraphSubscriptionRecord | null = null; - if (subscriptionStore) { - previousSubscription = subscriptionStore.load - ? await subscriptionStore.load(contextGraphId) - : (await subscriptionStore.loadAll()).find((row) => row.id === contextGraphId) ?? null; - } - - let membershipAttempted = false; - let subscriptionAttempted = false; - try { - // Membership is the prepare record; the subscription row is the durable - // activation/rehydration commit marker and must remain last. A legacy - // membership store without a read API may retain an idempotent prepared - // row after failure, but it can never leave a restart-visible active - // subscription without the membership fact it depends on. - if (membershipStore) membershipAttempted = true; - await this.upsertContextGraphMember(membership, { strict: true }); - if (subscriptionStore) subscriptionAttempted = true; - await this.persistContextGraphSubscriptionStrict( - contextGraphId, - subscription, - true, - ); - } catch (error) { - const rollbackFailures: unknown[] = []; - // A store may reject after applying a write, so compensate every - // attempted operation whose prior value was observable, including the - // one that surfaced the failure. Never guess absence for a legacy - // membership store without loadAll(): deleting there could erase a - // valid pre-existing row after an upsert that rejected before mutation. - if (subscriptionAttempted && subscriptionStore) { - try { - await this.enqueueContextGraphSubscriptionPersistWrite( - contextGraphId, - () => previousSubscription - ? subscriptionStore.save(previousSubscription) - : subscriptionStore.delete(contextGraphId), - ); - } catch (rollbackError) { - rollbackFailures.push(rollbackError); + let previousSubscription: ContextGraphSubscriptionRecord | null = null; + if (subscriptionStore) { + previousSubscription = subscriptionStore.load + ? await subscriptionStore.load(contextGraphId) + : (await subscriptionStore.loadAll()).find((row) => row.id === contextGraphId) ?? null; } - } - if (membershipAttempted && membershipStore && previousMembershipKnown) { + + const subscriptionRecord: ContextGraphSubscriptionRecord = { + id: contextGraphId, + name: subscription.name, + subscribed: subscription.subscribed, + synced: subscription.synced, + sharedMemorySynced: subscription.sharedMemorySynced, + metaSynced: subscription.metaSynced, + onChainId: subscription.onChainId, + onChainHash: subscription.onChainHash, + lastReconciledOrdinal: subscription.lastReconciledOrdinal, + coreHosted: subscription.coreHosted, + syncScoped: true, + }; + let membershipAttempted = false; + let subscriptionAttempted = false; try { - if (previousMembership) { - await membershipStore.upsert(previousMembership); - } else { - await membershipStore.delete( - contextGraphId, - membership.principalType, - normalizedPrincipalId, + if (membershipStore) { + membershipAttempted = true; + await membershipStore.upsert({ + ...membership, + principalId: normalizedPrincipalId, + updatedAt: Date.now(), + }); + } + if (subscriptionStore) { + subscriptionAttempted = true; + await subscriptionStore.save(subscriptionRecord); + } + } catch (error) { + const rollbackFailures: unknown[] = []; + if (subscriptionAttempted && subscriptionStore) { + try { + await (previousSubscription + ? subscriptionStore.save(previousSubscription) + : subscriptionStore.delete(contextGraphId)); + } catch (rollbackError) { + rollbackFailures.push(rollbackError); + } + } + if (membershipAttempted && membershipStore && previousMembershipKnown) { + try { + if (previousMembership) await membershipStore.upsert(previousMembership); + else { + await membershipStore.delete( + contextGraphId, + membership.principalType, + normalizedPrincipalId, + ); + } + } catch (rollbackError) { + rollbackFailures.push(rollbackError); + } + } + if (rollbackFailures.length > 0) { + const originalMessage = error instanceof Error ? error.message : String(error); + const rollbackMessage = rollbackFailures + .map((rollbackError) => rollbackError instanceof Error + ? rollbackError.message + : String(rollbackError)) + .join('; '); + throw new AggregateError( + [error, ...rollbackFailures], + `${originalMessage}; join-approval rollback failed: ${rollbackMessage}`, ); } - } catch (rollbackError) { - rollbackFailures.push(rollbackError); + throw error; } - } - if (rollbackFailures.length > 0) { - const originalMessage = error instanceof Error ? error.message : String(error); - const rollbackMessage = rollbackFailures - .map((rollbackError) => rollbackError instanceof Error - ? rollbackError.message - : String(rollbackError)) - .join('; '); - throw new AggregateError( - [error, ...rollbackFailures], - `${originalMessage}; join-approval rollback failed: ${rollbackMessage}`, - ); - } - throw error; - } + }), + { strict: true }, + ); } async assertAlreadyMemberDelegationRefresh(this: DKGAgent, @@ -7337,8 +7728,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { ...record, principalId: this.normalizeMembershipPrincipal(record.principalType, record.principalId), }; - const updatedAt = Date.now(); - const write = store.upsert({ ...normalizedRecord, updatedAt }); + const key = `${normalizedRecord.contextGraphId}\0${normalizedRecord.principalType}\0${normalizedRecord.principalId}`; + const write = this.enqueueContextGraphMembershipPersistWrite( + key, + () => store.upsert({ ...normalizedRecord, updatedAt: Date.now() }), + { strict: options?.strict === true }, + ); if (options?.strict === true) return write; // Background callers stay log-and-continue; durability-sensitive callers // opt into the strict path above and receive the original rejection. @@ -7358,7 +7753,11 @@ export class LifecycleSyncMethods extends DKGAgentBase { const store = this.config.contextGraphMembershipStore; if (!store) return; const normalizedPrincipalId = this.normalizeMembershipPrincipal(principalType, principalId); - void store.delete(contextGraphId, principalType, normalizedPrincipalId).catch((err) => { + const key = `${contextGraphId}\0${principalType}\0${normalizedPrincipalId}`; + void this.enqueueContextGraphMembershipPersistWrite( + key, + () => store.delete(contextGraphId, principalType, normalizedPrincipalId), + ).catch((err) => { this.log.warn( createOperationContext('system'), `Failed to delete context-graph membership for "${contextGraphId}" (${principalType}:${normalizedPrincipalId}): ${err instanceof Error ? err.message : String(err)}`, diff --git a/packages/agent/src/dkg-agent-ownership.ts b/packages/agent/src/dkg-agent-ownership.ts index 8779623b88..06dd4d6b63 100644 --- a/packages/agent/src/dkg-agent-ownership.ts +++ b/packages/agent/src/dkg-agent-ownership.ts @@ -669,7 +669,11 @@ export class OwnershipMethods extends DKGAgentBase { return false; } - async getContextGraphOwner(this: DKGAgent, contextGraphId: string): Promise { + async getContextGraphOwner( + this: DKGAgent, + contextGraphId: string, + options: { signal?: AbortSignal } = {}, + ): Promise { // Prefer the curator (wallet-scoped owner) so per-agent authorization // works on multi-agent nodes. Fall back to the creator (libp2p peer ID) // for legacy CGs created before the curator triple existed. @@ -681,9 +685,9 @@ export class OwnershipMethods extends DKGAgentBase { // that preference, the LIMIT-1 lookup here would non-deterministically // pick a foreign curator and lock the real local curator out of // manage-participants / rename / policy operations. - const curatorOwner = await this.getContextGraphCurator(contextGraphId); + const curatorOwner = await this.getContextGraphCurator(contextGraphId, options); if (curatorOwner) return curatorOwner; - const fromCreator = await this.getContextGraphCreator(contextGraphId); + const fromCreator = await this.getContextGraphCreator(contextGraphId, options); if (fromCreator) return fromCreator; // Final fallback: V10 wallet-scoped cgId convention (`0x.../`) // encodes the curator structurally, which lets us answer for CGs @@ -701,7 +705,7 @@ export class OwnershipMethods extends DKGAgentBase { // create stray `_meta` rows for graphs that were never created // here. The fallback is meant to rescue real-but-half-registered // graphs, not impersonate ownership of unknown ones. - const exists = await this.contextGraphExists(contextGraphId); + const exists = await this.contextGraphExists(contextGraphId, options); if (!exists) return null; return deriveCuratorDidFromCgId(contextGraphId); } @@ -858,8 +862,12 @@ export class OwnershipMethods extends DKGAgentBase { * Emitted approve/revoke binding metadata must use this value so remote * peers validating via `gossip-publish-handler` see a matching owner. */ - async getContextGraphCreator(this: DKGAgent, contextGraphId: string): Promise { - return (await this.getCgMeta(contextGraphId)).creator ?? null; + async getContextGraphCreator( + this: DKGAgent, + contextGraphId: string, + options: { signal?: AbortSignal } = {}, + ): Promise { + return (await this.getCgMeta(contextGraphId, { signal: options.signal })).creator ?? null; } public async listCclPolicyBindings(this: DKGAgent, opts: { diff --git a/packages/agent/src/dkg-agent-registry.ts b/packages/agent/src/dkg-agent-registry.ts index 153e401414..82f153092f 100644 --- a/packages/agent/src/dkg-agent-registry.ts +++ b/packages/agent/src/dkg-agent-registry.ts @@ -1484,8 +1484,12 @@ export class AgentRegistryMethods extends DKGAgentBase { * Check whether any locally registered agent is the curator/creator * of the given context graph. */ - async isCuratorOf(this: DKGAgent, contextGraphId: string): Promise { - const owner = await this.getContextGraphOwner(contextGraphId); + async isCuratorOf( + this: DKGAgent, + contextGraphId: string, + options: { signal?: AbortSignal } = {}, + ): Promise { + const owner = await this.getContextGraphOwner(contextGraphId, options); if (!owner) return false; // Mirror the comparison in PROTOCOL_JOIN_REQUEST. `normalizeAgentDid` // collapses EVM-address case drift but preserves peer-ID case. diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index 70d4a19fb2..f6a3f3040f 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -9,6 +9,7 @@ */ import { createHash, randomUUID } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; import { DKGNode, ProtocolRouter, GossipSubManager, TypedEventBus, DKGEvent, LibP2PNetwork, PeerResolver, StubNetworkStateRegistry, @@ -121,7 +122,7 @@ import { type WorkspaceAgentRecipientResolverInput, type WorkspaceSenderKeyEncryptInput, type SharedMemoryPublicSnapshotStorageConfig, type WorkspacePublicSnapshotStore, - readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, + readMaterializedVersion, shouldApplyMaterialization, withMaterializationLock, type MaterializedVersion, } from '@origintrail-official/dkg-publisher'; import { ethers } from 'ethers'; @@ -244,6 +245,7 @@ import { import { ContextGraphOnChainIdUnresolvedError, VmReconcileUnavailableError, + VmReconcileQueueClosedError, type ContextGraphReconcileResult, type VmReconcileSource, } from './vm-reconcile-service.js'; @@ -354,6 +356,7 @@ import { type ContextGraphSub, type ContextGraphSubscriptionRecord, type ContextGraphSubscriptionStore, + type VmReconcileRotationRecord, type ContextGraphMemberPrincipalType, type ContextGraphMemberStatus, type ContextGraphMembershipRecord, @@ -399,6 +402,11 @@ import { } from './dkg-agent-swm-state.js'; import { DKGAgentBase } from './dkg-agent-base.js'; import type { DKGAgent } from './dkg-agent.js'; +import { + bumpContextGraphBindingGeneration as bumpBindingGeneration, + captureContextGraphBindingGeneration as captureBindingGeneration, + isContextGraphBindingGenerationCurrent as isBindingGenerationCurrent, +} from './context-graph-binding-generation.js'; const DEFAULT_HOST_MODE_RECONCILE_BATCH_SIZE = 32; @@ -408,9 +416,15 @@ type VmReconcileTarget = { onChainId: string; onChainCgId: bigint; cursor: CursorState; + bindingGeneration: number; watermarkBefore: number; }; +type VmReconcileExecution = { + identityCursor: CursorState; + persistWatermark: (localCgId: string, watermark: number) => void; +}; + type VmReconcileOrdinalOptions = { /** Shared by every ordinal in one bounded pass. */ acquireActiveFetchPermit?: () => boolean; @@ -455,6 +469,25 @@ function stripBindingQuotes(v: string): string { return v; } +async function raceVmReconcileAbort( + work: Promise, + signal: AbortSignal | undefined, +): Promise { + if (!signal) return work; + void work.catch(() => undefined); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(new VmReconcileQueueClosedError()); + if (signal.aborted) onAbort(); + else signal.addEventListener('abort', onAbort, { once: true }); + }); + try { + return await Promise.race([work, aborted]); + } finally { + if (onAbort) signal.removeEventListener('abort', onAbort); + } +} + export class SwmHostModeMethods extends DKGAgentBase { /** * OT-RFC-38 LU-6 — initialize the on-disk opaque ciphertext store @@ -2276,6 +2309,26 @@ export class SwmHostModeMethods extends DKGAgentBase { return fallback; } + bumpContextGraphBindingGeneration(this: DKGAgent, localCgId: string): number { + return bumpBindingGeneration(this.contextGraphBindingGenerations, localCgId); + } + + captureContextGraphBindingGeneration(this: DKGAgent, localCgId: string): number { + return captureBindingGeneration(this.contextGraphBindingGenerations, localCgId); + } + + isContextGraphBindingGenerationCurrent( + this: DKGAgent, + localCgId: string, + generation: number, + ): boolean { + return isBindingGenerationCurrent( + this.contextGraphBindingGenerations, + localCgId, + generation, + ); + } + /** * Bind (or rebind) a local CG to an on-chain CG id, resetting the * chain-driven reconcile watermark if the bound id actually CHANGES. @@ -2290,8 +2343,10 @@ export class SwmHostModeMethods extends DKGAgentBase { */ bindSubscriptionOnChainId(this: DKGAgent, localCgId: string, sub: ContextGraphSub, newOnChainId: string): void { const prev = sub.onChainId; + if (prev === newOnChainId) return; + bumpBindingGeneration(this.contextGraphBindingGenerations, localCgId); sub.onChainId = newOnChainId; - if (!prev || prev === newOnChainId) return; + if (!prev) return; // The bound on-chain id actually CHANGED (repair / recreate / re-register). // Any prior reconcile progress refers to the OLD chain graph and must be // dropped, otherwise the sweep resumes at the wrong ordinal and skips @@ -2567,17 +2622,31 @@ export class SwmHostModeMethods extends DKGAgentBase { * burst of live nudges) collapse into one sweep per CG. */ async runVmReconcileSweep(this: DKGAgent): Promise { + if (this.started && !this.vmReconcileRuntimeReady) return; + const lifecycleGeneration = this.vmReconcileLifecycleGeneration; + const lifecycleSignal = this.vmReconcileLifecycleController?.signal; + const isLifecycleCurrent = () => !this.vmReconcileRotationClosed + && !lifecycleSignal?.aborted + && this.vmReconcileLifecycleGeneration === lifecycleGeneration; const dispatcher = this.vmReconcileDispatcher; - if (!this.vmReconcileEnabled() || !dispatcher) return; + if (!isLifecycleCurrent() || !this.vmReconcileEnabled() || !dispatcher) return; if (this.vmReconcileSweepInFlight) return this.vmReconcileSweepInFlight; const running = (async () => { const eligible: string[] = []; for (const [localCgId, sub] of this.subscribedContextGraphs) { + if (!isLifecycleCurrent()) return; // GH #1098 — self-prime onChainId for a pre-subscribed PUBLIC member CG // (subscribed BEFORE its first publish, so unbound) before the skip-gate // below would pass it over. Shared with the live KACG nudge. if (sub.subscribed && !sub.onChainId) { - await this.selfPrimeSubscriptionOnChainId(localCgId, sub); + await this.selfPrimeSubscriptionOnChainId( + localCgId, + sub, + undefined, + isLifecycleCurrent, + lifecycleSignal, + ); + if (!isLifecycleCurrent()) return; } // Member subscriptions AND Phase D core-hosted public CGs get swept. if ((!sub.subscribed && !sub.coreHosted) || !sub.onChainId) continue; @@ -2594,9 +2663,11 @@ export class SwmHostModeMethods extends DKGAgentBase { // foreground live/manual work can still enter the unified dispatcher. const start = this.vmReconcileSweepCursor % eligible.length; for (let offset = 0; offset < eligible.length; offset += 1) { + if (!isLifecycleCurrent()) return; const index = (start + offset) % eligible.length; await dispatcher.dispatch(eligible[index]!, 'periodic').catch(() => undefined); } + if (!isLifecycleCurrent()) return; this.vmReconcileSweepCursor = (start + 1) % eligible.length; })(); this.vmReconcileSweepInFlight = running; @@ -2627,24 +2698,54 @@ export class SwmHostModeMethods extends DKGAgentBase { localCgId: string, sub: ContextGraphSub, targetOnChainId?: bigint, + isCurrent: () => boolean = () => true, + signal?: AbortSignal, ): Promise { - if (!sub.subscribed || sub.onChainId) return null; + const bindingGeneration = captureBindingGeneration( + this.contextGraphBindingGenerations, + localCgId, + ); + const isSubscriptionCurrent = () => isCurrent() + && this.subscribedContextGraphs.get(localCgId) === sub + && sub.subscribed + && !sub.onChainId + && isBindingGenerationCurrent( + this.contextGraphBindingGenerations, + localCgId, + bindingGeneration, + ); + if (!isSubscriptionCurrent()) return null; let resolved: string | null = null; try { - resolved = await this.getContextGraphOnChainId(localCgId, { - source: 'agent.vmReconcile.resolveOnChainId', - }); + resolved = await raceVmReconcileAbort( + this.getContextGraphOnChainId(localCgId, { + signal, + source: 'agent.vmReconcile.resolveOnChainId', + }), + signal, + ); } catch { return null; } - if (!resolved) return null; + if (!isSubscriptionCurrent() || !resolved) return null; if (targetOnChainId !== undefined) { let resolvedNum: bigint | null = null; try { resolvedNum = BigInt(resolved); } catch { return null; } if (resolvedNum !== targetOnChainId) return null; } + if (!isSubscriptionCurrent()) return null; + try { + await this.persistContextGraphSubscriptionStrict( + localCgId, + { ...sub, onChainId: resolved }, + undefined, + isSubscriptionCurrent, + ); + } catch { + return null; + } + if (!isSubscriptionCurrent()) return null; this.bindSubscriptionOnChainId(localCgId, sub, resolved); - this.persistContextGraphSubscription(localCgId); return resolved; } @@ -2672,6 +2773,12 @@ export class SwmHostModeMethods extends DKGAgentBase { kaId: bigint, ctx: OperationContext, ): Promise { + const lifecycleGeneration = this.vmReconcileLifecycleGeneration; + const lifecycleSignal = this.vmReconcileLifecycleController?.signal; + const isLifecycleCurrent = () => !this.vmReconcileRotationClosed + && !lifecycleSignal?.aborted + && this.vmReconcileLifecycleGeneration === lifecycleGeneration; + if (!isLifecycleCurrent()) return null; let targetOnChain: bigint | null = null; try { targetOnChain = BigInt(onChainId); } catch { targetOnChain = null; } @@ -2684,10 +2791,20 @@ export class SwmHostModeMethods extends DKGAgentBase { // path); the sweep remains the safety net for a CG whose quad hasn't arrived. if (targetOnChain !== null) { for (const [lcg, sub] of this.subscribedContextGraphs) { - const bound = await this.selfPrimeSubscriptionOnChainId(lcg, sub, targetOnChain); + if (!isLifecycleCurrent()) return null; + const bound = await this.selfPrimeSubscriptionOnChainId( + lcg, + sub, + targetOnChain, + isLifecycleCurrent, + lifecycleSignal, + ); + if (!isLifecycleCurrent()) return null; if (bound) { this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> bound + reconcile pre-subscribed "${lcg}"`); - if (this.vmReconcileDispatcher) void this.vmReconcileDispatcher.triggerLive(lcg); + if (this.vmReconcileDispatcher && isLifecycleCurrent()) { + void this.vmReconcileDispatcher.triggerLive(lcg); + } return lcg; } } @@ -2698,9 +2815,11 @@ export class SwmHostModeMethods extends DKGAgentBase { const sub = this.subscribedContextGraphs.get(localCgId); // Populate VM for CGs we member-subscribe to OR (Phase D) public CGs this // Core hosts — a hosted Core fills its own gaps too. - if (!sub?.subscribed && !sub?.coreHosted) return null; + if (!isLifecycleCurrent() || (!sub?.subscribed && !sub?.coreHosted)) return null; this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> reconcile "${localCgId}"`); - if (this.vmReconcileDispatcher) void this.vmReconcileDispatcher.triggerLive(localCgId); + if (this.vmReconcileDispatcher && isLifecycleCurrent()) { + void this.vmReconcileDispatcher.triggerLive(localCgId); + } return localCgId; } @@ -2716,12 +2835,18 @@ export class SwmHostModeMethods extends DKGAgentBase { localCgId: string, source: VmReconcileSource = 'manual', ): Promise { + if (this.started && !this.vmReconcileRuntimeReady) { + throw new VmReconcileQueueClosedError(); + } return this.ensureVmReconcileDispatcher().dispatch(localCgId, source); } ensureVmReconcileDispatcher( this: DKGAgent, ): VmReconcileDispatcher { + if (this.started && !this.vmReconcileRuntimeReady) { + throw new VmReconcileQueueClosedError(); + } if (!this.vmReconcileDispatcher) { this.vmReconcileDispatcher = new VmReconcileDispatcher( (localCgId, source) => this.executeVmReconcileForCg(localCgId, source), @@ -2746,39 +2871,119 @@ export class SwmHostModeMethods extends DKGAgentBase { localCgId: string, source: VmReconcileSource, ): Promise { - const target = await this.resolveVmReconcileTarget(localCgId); - - // Keep the legacy-label -> scoped-VM migration in the admitted lane and - // before the evidence gate. A current watermark can still need this repair. - await this.healStrandedScopedKCs(localCgId, target.sub); - - const result = await reconcileContextGraph( - this.createVmReconcileDeps(localCgId), - target.cursor, - localCgId, - target.onChainCgId, - ); - const response = this.toContextGraphReconcileResult(localCgId, source, target, result); - this.emitVmReconcileTelemetry(localCgId, target, result, response.status); - // Queue one trailing slice while this key is still active. The dispatcher - // places it behind already-waiting live CGs, so a large graph makes steady - // progress without monopolising the only VM worker. - if (result.hasMore || result.staleTarget) { - this.vmReconcileDispatcher?.triggerLive(localCgId); - } - return response; + const lifecycleGeneration = this.vmReconcileLifecycleGeneration; + const lifecycleSignal = this.vmReconcileLifecycleController?.signal; + const isLifecycleCurrent = () => !this.vmReconcileRotationClosed + && !lifecycleSignal?.aborted + && this.vmReconcileLifecycleGeneration === lifecycleGeneration; + if (!isLifecycleCurrent()) throw new VmReconcileQueueClosedError(); + const physicalRun = (async (): Promise => { + const target = await this.resolveVmReconcileTarget( + localCgId, + isLifecycleCurrent, + lifecycleSignal, + ); + const isTargetCurrent = () => isLifecycleCurrent() + && this.subscribedContextGraphs.get(localCgId) === target.sub + && target.sub.onChainId === target.onChainId + && isBindingGenerationCurrent( + this.contextGraphBindingGenerations, + localCgId, + target.bindingGeneration, + ) + && this.reconcileCursors.get(localCgId) === target.cursor; + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); + + // Keep the legacy-label -> scoped-VM migration in the admitted lane and + // before the evidence gate. A current watermark can still need this repair. + await this.healStrandedScopedKCs( + localCgId, + target.sub, + isTargetCurrent, + lifecycleSignal, + ); + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); + + // Reconcile on a private cursor snapshot. The caller-facing abort race may + // finish before an adapter physically settles; a stale continuation must + // never mutate the live cursor or persist a watermark into a new binding. + const workingCursor: CursorState = { + watermark: target.cursor.watermark, + ahead: new Map(target.cursor.ahead), + scanOrdinal: target.cursor.scanOrdinal, + }; + let pendingWatermark: number | undefined; + const result = await reconcileContextGraph( + this.createVmReconcileDeps( + localCgId, + lifecycleGeneration, + target, + lifecycleSignal, + { + identityCursor: target.cursor, + persistWatermark: (_lcg, watermark) => { pendingWatermark = watermark; }, + }, + ), + workingCursor, + localCgId, + target.onChainCgId, + ); + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); + if (result.reconciled > 0 || pendingWatermark !== undefined) { + await this.store.flush?.({ + priority: 'background', + source: 'agent.vmReconcile.materialization.flush', + }); + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); + } + if (pendingWatermark !== undefined) { + await this.persistVmReconcileWatermark( + localCgId, + pendingWatermark, + target.sub, + target.bindingGeneration, + target.cursor, + ); + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); + } + target.cursor.watermark = workingCursor.watermark; + target.cursor.ahead = new Map(workingCursor.ahead); + target.cursor.scanOrdinal = workingCursor.scanOrdinal; + const response = this.toContextGraphReconcileResult(localCgId, source, target, result); + this.emitVmReconcileTelemetry(localCgId, target, result, response.status); + // Queue one trailing slice while this key is still active. The dispatcher + // places it behind already-waiting live CGs, so a large graph makes steady + // progress without monopolising the only VM worker. + if (isLifecycleCurrent() && (result.hasMore || result.staleTarget)) { + this.vmReconcileDispatcher?.triggerLive(localCgId); + } + return response; + })(); + this.vmReconcilePhysicalRuns.add(physicalRun); + void physicalRun.finally(() => { + this.vmReconcilePhysicalRuns.delete(physicalRun); + }).catch(() => undefined); + return raceVmReconcileAbort(physicalRun, lifecycleSignal); } async resolveVmReconcileTarget( this: DKGAgent, localCgId: string, + isCurrent: () => boolean = () => true, + signal?: AbortSignal, ): Promise { let sub = this.subscribedContextGraphs.get(localCgId); if (!sub?.subscribed && !sub?.coreHosted) { throw new ContextGraphNotFoundError(localCgId); } if (!sub.onChainId && sub.subscribed) { - await this.selfPrimeSubscriptionOnChainId(localCgId, sub); + await this.selfPrimeSubscriptionOnChainId( + localCgId, + sub, + undefined, + isCurrent, + signal, + ); sub = this.subscribedContextGraphs.get(localCgId); } if (!sub?.onChainId) { @@ -2798,23 +3003,46 @@ export class SwmHostModeMethods extends DKGAgentBase { onChainId: sub.onChainId, onChainCgId: BigInt(sub.onChainId), cursor, + bindingGeneration: captureBindingGeneration( + this.contextGraphBindingGenerations, + localCgId, + ), watermarkBefore: cursor.watermark, }; } - createVmReconcileDeps(this: DKGAgent, localCgId: string): ChainReconcilerDeps { - const capturedSub = this.subscribedContextGraphs.get(localCgId); - const capturedOnChainId = capturedSub?.onChainId; - const capturedCursor = this.reconcileCursors.get(localCgId); + createVmReconcileDeps( + this: DKGAgent, + localCgId: string, + lifecycleGeneration = this.vmReconcileLifecycleGeneration, + target?: VmReconcileTarget, + signal?: AbortSignal, + execution?: VmReconcileExecution, + ): ChainReconcilerDeps { + const capturedSub = target?.sub ?? this.subscribedContextGraphs.get(localCgId); + const capturedOnChainId = target?.onChainId ?? capturedSub?.onChainId; + const capturedBindingGeneration = target?.bindingGeneration + ?? captureBindingGeneration(this.contextGraphBindingGenerations, localCgId); + const capturedCursor = execution?.identityCursor + ?? target?.cursor + ?? this.reconcileCursors.get(localCgId); const isTargetCurrent = (): boolean => { const current = this.subscribedContextGraphs.get(localCgId); - return current === capturedSub + return !this.vmReconcileRotationClosed + && this.vmReconcileLifecycleGeneration === lifecycleGeneration + && current === capturedSub && current?.onChainId === capturedOnChainId + && isBindingGenerationCurrent( + this.contextGraphBindingGenerations, + localCgId, + capturedBindingGeneration, + ) && this.reconcileCursors.get(localCgId) === capturedCursor; }; return { getKCCount: async (cg) => { const head = Number(await this.chain.getContextGraphKCCount!(cg)); + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); if (!Number.isSafeInteger(head) || head < 0) { throw new Error(`Invalid on-chain KC count for context graph "${localCgId}": ${head}`); } @@ -2824,7 +3052,9 @@ export class SwmHostModeMethods extends DKGAgentBase { // Capability-absent chains disable the reorg gate; transient RPC // failures still throw so the durable watermark cannot advance. if (typeof this.chain.getBlockNumber !== 'function') return undefined; - return await this.chain.getBlockNumber(); + const headBlock = await this.chain.getBlockNumber(); + if (!isTargetCurrent()) throw new VmReconcileQueueClosedError(); + return headBlock; }, reconcileOrdinal: (lcg, ocg, ordinal, headBlock) => this.reconcileChainOrdinal(lcg, ocg, ordinal, headBlock, { @@ -2832,29 +3062,73 @@ export class SwmHostModeMethods extends DKGAgentBase { deferActiveFetch: true, }), recoverPendingOrdinals: (lcg, ocg, targets, headBlock) => - this.recoverVmReconcileBatch(lcg, ocg, targets, headBlock, isTargetCurrent), + this.recoverVmReconcileBatch( + lcg, + ocg, + targets, + headBlock, + isTargetCurrent, + signal, + ), maxOrdinalsPerPass: DKGAgentBase.VM_RECONCILE_BATCH_SIZE, maxOrdinalConcurrency: DKGAgentBase.VM_RECONCILE_ORDINAL_CONCURRENCY, isTargetCurrent: () => isTargetCurrent(), persistWatermark: (lcg, watermark) => { - const sub = this.subscribedContextGraphs.get(lcg); - if (!sub) return; - const previous = sub.lastReconciledOrdinal ?? 0; - sub.lastReconciledOrdinal = watermark; - this.persistContextGraphSubscription(lcg); - this.emitReplication({ - contextGraphId: lcg, - onChainCgId: sub.onChainId, - action: 'cursor-advance', - fromWatermark: previous, - toWatermark: watermark, - }); + if (!isTargetCurrent()) return; + if (execution) execution.persistWatermark(lcg, watermark); + else void this.persistVmReconcileWatermark( + lcg, + watermark, + capturedSub, + capturedBindingGeneration, + capturedCursor, + ); }, confirmationDepth: DKGAgentBase.VM_RECONCILE_CONFIRMATION_DEPTH, log: (msg) => this.log.info(createOperationContext('system'), msg), }; } + persistVmReconcileWatermark( + this: DKGAgent, + localCgId: string, + watermark: number, + expectedSub?: ContextGraphSub, + expectedBindingGeneration = captureBindingGeneration( + this.contextGraphBindingGenerations, + localCgId, + ), + expectedCursor?: CursorState, + ): Promise { + const sub = this.subscribedContextGraphs.get(localCgId); + const isTargetCurrent = () => this.subscribedContextGraphs.get(localCgId) === sub + && (!expectedSub || sub === expectedSub) + && isBindingGenerationCurrent( + this.contextGraphBindingGenerations, + localCgId, + expectedBindingGeneration, + ) + && (!expectedCursor || this.reconcileCursors.get(localCgId) === expectedCursor); + if (!sub || !isTargetCurrent()) return Promise.resolve(); + const previous = sub.lastReconciledOrdinal ?? 0; + return this.persistContextGraphSubscriptionStrict( + localCgId, + { ...sub, lastReconciledOrdinal: watermark }, + undefined, + isTargetCurrent, + ).then(() => { + if (!isTargetCurrent()) return; + sub.lastReconciledOrdinal = watermark; + this.emitReplication({ + contextGraphId: localCgId, + onChainCgId: sub.onChainId, + action: 'cursor-advance', + fromWatermark: previous, + toWatermark: watermark, + }); + }); + } + toContextGraphReconcileResult( this: DKGAgent, localCgId: string, @@ -2945,9 +3219,20 @@ export class SwmHostModeMethods extends DKGAgentBase { * writer cannot clobber. NEVER calls `isAlreadyConfirmed` — that read-both * guard is the permanence mechanism that made the legacy promotion stick. */ - async healStrandedScopedKCs(this: DKGAgent, localCgId: string, sub: ContextGraphSub): Promise { + async healStrandedScopedKCs( + this: DKGAgent, + localCgId: string, + sub: ContextGraphSub, + isCurrent: () => boolean = () => true, + signal?: AbortSignal, + ): Promise { try { - if (!sub.onChainId) return; + const capturedOnChainId = sub.onChainId; + const canApply = () => isCurrent() + && (!(this.subscribedContextGraphs instanceof Map) + || this.subscribedContextGraphs.get(localCgId) === sub) + && sub.onChainId === capturedOnChainId; + if (!canApply() || !capturedOnChainId) return; // Server-side byte-safe copy is the ONLY safe relocation mechanism; if the // backend can't do SPARQL UPDATE we bail rather than risk a lossy JS round-trip. if (typeof this.store.update !== 'function') return; @@ -2962,16 +3247,16 @@ export class SwmHostModeMethods extends DKGAgentBase { this.store, sparql, touchedGraphs, - { source: 'agent.swm.rsHeal.materialize' }, + { signal, source: 'agent.swm.rsHeal.materialize' }, ); if (!updated) throw new Error('RS heal requires server-side update() support'); }; const DKG = 'http://dkg.io/ontology/'; const legacyMeta = contextGraphMetaUri(localCgId); - const scopedMeta = contextGraphMetaUri(localCgId, sub.onChainId); + const scopedMeta = contextGraphMetaUri(localCgId, capturedOnChainId); const rootData = contextGraphDataUri(localCgId); - const scopedData = contextGraphDataUri(localCgId, sub.onChainId); + const scopedData = contextGraphDataUri(localCgId, capturedOnChainId); // The publisher's OWN one-shot publish() writes confirmed PUBLIC data to a // per-KA verifiable-memory (VM) graph — NOT the legacy root data graph (the // receiver/#1259 strand fills root data). So the data reads below look in @@ -2980,29 +3265,38 @@ export class SwmHostModeMethods extends DKGAgentBase { // the receiver heal; prefix-scanning every VM graph could pull a DIFFERENT // KA's triples for a root IRI that recurs across per-KA VM graphs. - // 2a ASK-guard: is there at least one legacy-only KC (batchId present in - // legacy meta, absent in scoped meta)? In steady state this is one ASK per - // bound CG and returns false once healed. + // 2a ASK-guard: is there at least one incomplete scoped KC? Requiring the + // batch id and materialization version together makes a legacy partial + // write retryable instead of permanently hiding it from future sweeps. const askGuard = await this.store.query( `ASK { GRAPH <${legacyMeta}> { ?ual <${DKG}batchId> ?b } - FILTER NOT EXISTS { GRAPH <${scopedMeta}> { ?ual <${DKG}batchId> ?b } } + FILTER NOT EXISTS { + GRAPH <${scopedMeta}> { + ?ual <${DKG}batchId> ?b ; <${DKG}materializedVersion> ?version + } + } }`, - { source: 'agent.swm.rsHeal.findLegacyOnly' }, + { signal, source: 'agent.swm.rsHeal.findLegacyOnly' }, ); - if (askGuard.type !== 'boolean' || !askGuard.value) return; + if (!canApply() || askGuard.type !== 'boolean' || !askGuard.value) return; // 2b: enumerate the stranded UALs. const stranded = await this.store.query( `SELECT ?ual ?b WHERE { GRAPH <${legacyMeta}> { ?ual <${DKG}batchId> ?b } - FILTER NOT EXISTS { GRAPH <${scopedMeta}> { ?ual <${DKG}batchId> ?b } } + FILTER NOT EXISTS { + GRAPH <${scopedMeta}> { + ?ual <${DKG}batchId> ?b ; <${DKG}materializedVersion> ?version + } + } }`, - { source: 'agent.swm.rsHeal.listLegacyOnly' }, + { signal, source: 'agent.swm.rsHeal.listLegacyOnly' }, ); - if (stranded.type !== 'bindings') return; + if (!canApply() || stranded.type !== 'bindings') return; for (const row of stranded.bindings) { + if (!canApply()) return; // Bindings come back stripped to bare values by the store adapters // (oxigraph/sparql-http both emit IRIs unwrapped); strip + validate // exactly as the extractor does for its `ual`. @@ -3026,6 +3320,7 @@ export class SwmHostModeMethods extends DKGAgentBase { ); try { await withMaterializationLock(scopedMeta, ual, async () => { + if (!canApply()) return; // A KC may carry no `dkg:materializedVersion` stamp in legacy meta: // the publisher's OWN one-shot publish writes the KC into the legacy // label `_meta` but never stamps a version (only the @@ -3038,7 +3333,9 @@ export class SwmHostModeMethods extends DKGAgentBase { // reverse, so it can never clobber a genuine update. const version = (await readMaterializedVersion(this.store, legacyMeta, ual)) ?? { blockNumber: 0, txIndex: 0 }; + if (!canApply()) return; if (!(await shouldApplyMaterialization(this.store, scopedMeta, ual, version))) return; // idempotent + if (!canApply()) return; assertSafeIri(ual); @@ -3051,9 +3348,9 @@ export class SwmHostModeMethods extends DKGAgentBase { { <${ual}> <${DKG}rootEntity> ?root . } } }`, - { source: 'agent.swm.rsHeal.readRoots' }, + { signal, source: 'agent.swm.rsHeal.readRoots' }, ); - if (rootsRes.type !== 'bindings') return; + if (!canApply() || rootsRes.type !== 'bindings') return; const roots: string[] = []; const seen = new Set(); for (const r of rootsRes.bindings) { @@ -3085,9 +3382,9 @@ export class SwmHostModeMethods extends DKGAgentBase { } } }`, - { source: 'agent.swm.rsHeal.checkRootData' }, + { signal, source: 'agent.swm.rsHeal.checkRootData' }, ); - if (present.type !== 'boolean' || !present.value) return; + if (!canApply() || present.type !== 'boolean' || !present.value) return; } // DATA copy (per root) — MANDATORY server-side, byte-safe, read-both @@ -3095,6 +3392,7 @@ export class SwmHostModeMethods extends DKGAgentBase { // trustLevel stamps in BOTH branches so the recomputed leaf set stays // bit-identical with the on-chain merkleLeafCount. for (const root of roots) { + if (!canApply()) return; await update( `INSERT { GRAPH <${scopedData}> { ?s ?p ?o } } WHERE { { @@ -3113,33 +3411,57 @@ export class SwmHostModeMethods extends DKGAgentBase { }`, [scopedData], ); + if (!canApply()) return; } - // META copy — server-side. Carries the `` subject (batchId - // discriminator) + the `/` token rows - // (rootEntity/privateMerkleRoot). + // `update()` is not an atomic transaction on every supported HTTP + // store. Remove the completion marker first, copy metadata without + // that marker, and stamp completion only after the copy succeeds. + // Any partial copy therefore remains visible to the next heal. + if (!canApply()) return; + await update( + `DELETE WHERE { + GRAPH <${scopedMeta}> { + <${ual}> <${DKG}materializedVersion> ?oldVersion + } + }`, + [scopedMeta], + ); + if (!canApply()) return; await update( - `INSERT { GRAPH <${scopedMeta}> { ?s ?p ?o } } WHERE { + `INSERT { + GRAPH <${scopedMeta}> { ?s ?p ?o } + } + WHERE { GRAPH <${legacyMeta}> { ?s ?p ?o . FILTER(?s = <${ual}> || STRSTARTS(STR(?s), "${ual}/")) + FILTER(?p != <${DKG}materializedVersion>) } }`, [scopedMeta], ); - - // Stamp so a later stale writer can't clobber. - await writeMaterializedVersion(this.store, scopedMeta, ual, version); - - this.log.info( - createOperationContext('system'), - `RS heal: relocated stranded legacy KC ${ual} -> scoped cg=${sub.onChainId} (${roots.length} root(s))`, + if (!canApply()) return; + await update( + `INSERT DATA { + GRAPH <${scopedMeta}> { + <${ual}> <${DKG}materializedVersion> "${version.blockNumber}:${version.txIndex}" + } + }`, + [scopedMeta], ); - }); + + if (canApply()) { + this.log.info( + createOperationContext('system'), + `RS heal: relocated stranded legacy KC ${ual} -> scoped cg=${capturedOnChainId} (${roots.length} root(s))`, + ); + } + }, { signal }); } catch (err) { this.log.warn( createOperationContext('system'), - `RS heal: relocate failed for ${ual} (cg=${sub.onChainId}): ${err instanceof Error ? err.message : String(err)}`, + `RS heal: relocate failed for ${ual} (cg=${capturedOnChainId}): ${err instanceof Error ? err.message : String(err)}`, ); } } @@ -3372,7 +3694,10 @@ export class SwmHostModeMethods extends DKGAgentBase { deleteVmReconcileNegativeCacheEntry(this: DKGAgent, cacheKey: string): void { const existing = this.vmReconcileNegativeCache.get(cacheKey); - this.vmReconcileNegativeCacheHydrated.add(cacheKey); + this.markVmReconcileNegativeCacheHydrated( + cacheKey, + existing?.localCgId ?? cacheKey.slice(0, Math.max(0, cacheKey.indexOf('\0'))), + ); if (existing) { this.vmReconcileNegativeCache.delete(cacheKey); const keys = this.vmReconcileNegativeCacheKeysByCg.get(existing.localCgId); @@ -3395,13 +3720,30 @@ export class SwmHostModeMethods extends DKGAgentBase { keys.add(cacheKey); } + markVmReconcileNegativeCacheHydrated(this: DKGAgent, cacheKey: string, localCgId: string): void { + // Access order keeps actively reused keys resident while old one-shot + // misses fall out. Eviction is fail-open: it only permits another durable + // lookup if the same key is encountered later. + this.vmReconcileNegativeCacheHydrated.delete(cacheKey); + this.vmReconcileNegativeCacheHydrated.set(cacheKey, localCgId); + while ( + this.vmReconcileNegativeCacheHydrated.size + > DKGAgentBase.VM_RECONCILE_CACHE_MAX_ENTRIES + ) { + const oldestKey = this.vmReconcileNegativeCacheHydrated.keys().next().value; + if (oldestKey === undefined) break; + this.vmReconcileNegativeCacheHydrated.delete(oldestKey); + } + } + async shouldDeferVmReconcileByNegativeCache(this: DKGAgent, cacheKey: string, localCgId: string, ): Promise { let cached = this.vmReconcileNegativeCache.get(cacheKey); - if (!cached && !this.vmReconcileNegativeCacheHydrated.has(cacheKey)) { - this.vmReconcileNegativeCacheHydrated.add(cacheKey); + const durableStateAlreadyConsulted = this.vmReconcileNegativeCacheHydrated.has(cacheKey); + this.markVmReconcileNegativeCacheHydrated(cacheKey, localCgId); + if (!cached && !durableStateAlreadyConsulted) { try { const durable = await this.config.contextGraphSubscriptionStore ?.loadVmReconcileNegative?.(cacheKey); @@ -3535,7 +3877,7 @@ export class SwmHostModeMethods extends DKGAgentBase { peerTopologyKey: state.peerTopologyKey, }; this.vmReconcileNegativeCache.set(cacheKey, record); - this.vmReconcileNegativeCacheHydrated.add(cacheKey); + this.markVmReconcileNegativeCacheHydrated(cacheKey, localCgId); this.indexVmReconcileNegativeCacheEntry(localCgId, cacheKey); const durableStore = this.config.contextGraphSubscriptionStore; if (this.vmReconcileSwmGenSupportsDurableNegative(record.swmGen)) { @@ -3553,6 +3895,460 @@ export class SwmHostModeMethods extends DKGAgentBase { this.pruneVmReconcileState(); } + vmReconcileRotationNow(this: DKGAgent): number { + return performance.now(); + } + + vmReconcileRotationSlotKey( + this: DKGAgent, + target: OrdinalRecoveryTarget, + ): string { + return `${target.localCgId}\0${target.onChainCgId}\0${target.ordinal}`; + } + + clearVmReconcileRotationStateForSlot( + this: DKGAgent, + localCgId: string, + onChainCgId: bigint, + ordinal: number, + ): void { + this.vmReconcileRotationState.delete(`${localCgId}\0${onChainCgId.toString()}\0${ordinal}`); + } + + vmReconcileRotationFingerprint( + this: DKGAgent, + target: OrdinalRecoveryTarget, + ): string { + return `${target.ual}\0${target.merkleRoot.toLowerCase()}`; + } + + vmReconcileObservedCandidatePeerIds( + this: DKGAgent, + localCgId: string, + ): string[] { + const curatorOrder = this.vmReconcileCuratorPeersByCg.get(localCgId) ?? []; + const libp2p = (this.node as any)?.libp2p; + const getConnections = libp2p?.getConnections; + if (typeof getConnections !== 'function') { + return curatorOrder.slice(0, DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX); + } + const peersById = new Map(); + for (const connection of getConnections.call(libp2p) as Array<{ + remotePeer?: { toString(): string }; + }>) { + const peer = connection.remotePeer; + const peerId = peer?.toString(); + if (!peer || !peerId || peerId === this.peerId || peersById.has(peerId)) continue; + peersById.set(peerId, peer); + } + // `getConnections()` iteration order is transport state, not candidate + // identity. Sort before tiering/capping so harmless connection reorder can + // never replace one member of the bounded proof roster. + const canonicalPeers = [...peersById.values()].sort((left, right) => + left.toString().localeCompare(right.toString())); + const ordinaryOrder = this.selectCatchupPeers( + canonicalPeers, + this.preferredSyncPeers.get(localCgId), + false, + ) + .map((peer) => peer.toString()); + // Structural curators are authoritative and may contain the only holder. + // Ordinary connected peers are opportunistic fallback only: cap that tier + // separately so connection churn cannot stretch a negative cycle to 256 + // elevated prefix downloads. + const boundedCurators = [...new Set(curatorOrder)] + .slice(0, DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX); + const curatorSet = new Set(boundedCurators); + const ordinaryBudget = Math.min( + DKGAgentBase.VM_RECONCILE_EXACT_PEER_MAX, + Math.max(0, DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX - boundedCurators.length), + ); + const boundedOrdinary = ordinaryOrder + .filter((peerId) => !curatorSet.has(peerId)) + .slice(0, ordinaryBudget); + return [...boundedCurators, ...boundedOrdinary]; + } + + vmReconcilePeerMembershipMatches( + this: DKGAgent, + left: ReadonlySet, + right: readonly string[], + ): boolean { + return left.size === right.length && right.every((peerId) => left.has(peerId)); + } + + touchVmReconcileRotationRecord( + this: DKGAgent, + slotKey: string, + record: VmReconcileRotationRecord, + ): void { + if (this.vmReconcileRotationState.get(slotKey) !== record) return; + this.vmReconcileRotationState.delete(slotKey); + this.vmReconcileRotationState.set(slotKey, record); + } + + prepareVmReconcileRotationTarget( + this: DKGAgent, + target: OrdinalRecoveryTarget, + candidatePeerIds: readonly string[], + now: number, + curatorRosterConfirmed = true, + ): { + slotKey: string; + record?: VmReconcileRotationRecord; + suppressed: boolean; + } { + const slotKey = this.vmReconcileRotationSlotKey(target); + if (this.vmReconcileRotationClosed) return { slotKey, suppressed: true }; + + const fingerprint = this.vmReconcileRotationFingerprint(target); + let record = this.vmReconcileRotationState.get(slotKey); + if (record && record.fingerprint !== fingerprint) { + this.vmReconcileRotationState.delete(slotKey); + record = undefined; + } + if ( + record?.phase === 'backoff' + && record.backoffKind === 'clean-absence' + && (!record.curatorRosterConfirmed || !curatorRosterConfirmed) + ) { + // Absence gathered while curator discovery was unavailable must not + // suppress the next lookup: that lookup may reveal the only holder. + this.vmReconcileRotationState.delete(slotKey); + record = undefined; + } + if (candidatePeerIds.length === 0) { + // A transient empty socket view cannot invalidate a completed proof: doing + // so would redial and refetch every sweep after ordinary disconnects. + // Partial evidence is different and remains fail-open; drop it so the next + // non-empty roster starts a genuinely fresh cycle. + if (record?.phase === 'backoff' && now < record.nextRetryAt) { + this.touchVmReconcileRotationRecord(slotKey, record); + return { slotKey, record, suppressed: true }; + } + this.vmReconcileRotationState.delete(slotKey); + return { slotKey, suppressed: false }; + } + + if (!record) { + const nextRecord: VmReconcileRotationRecord = { + localCgId: target.localCgId, + onChainCgId: target.onChainCgId, + ordinal: target.ordinal, + fingerprint, + phase: 'collecting', + candidatePeerIds: new Set(candidatePeerIds), + attemptedPeerIds: new Set(), + cleanAbsentPeerIds: new Set(), + curatorRosterConfirmed, + collectionDeadlineAt: now + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS, + failures: 0, + nextRetryAt: 0, + }; + if (!this.installVmReconcileRotationRecord(slotKey, nextRecord)) { + // Preserve the pressure bound at cap: an unowned target cannot retain + // exponential retry state, so running elevated exact transport here + // would replay it every sweep. Defer until an expired/resolved slot is + // available; this is process-local scheduling, never absence evidence. + return { slotKey, suppressed: true }; + } + return { + slotKey, + record: nextRecord, + suppressed: false, + }; + } + + const membershipUnchanged = this.vmReconcilePeerMembershipMatches( + record.candidatePeerIds, + candidatePeerIds, + ); + const rosterProofUpgraded = !record.curatorRosterConfirmed && curatorRosterConfirmed; + if (!membershipUnchanged) { + const previousCandidatePeerIds = record.candidatePeerIds; + const nextCandidatePeerIds = new Set(candidatePeerIds); + record.candidatePeerIds = new Set(candidatePeerIds); + record.curatorRosterConfirmed = curatorRosterConfirmed; + const removedPeer = [...previousCandidatePeerIds] + .some((peerId) => !nextCandidatePeerIds.has(peerId)); + if (removedPeer) { + // A proof roster is a set, not an accumulation of surviving credits. + // Any removal/replacement invalidates the whole cycle so shrink can + // never manufacture exhaustion or preserve an active suppression. + record.phase = 'collecting'; + record.backoffKind = undefined; + record.nextRetryAt = 0; + record.attemptedPeerIds.clear(); + record.cleanAbsentPeerIds.clear(); + record.lastAttemptedPeerId = undefined; + record.collectionDeadlineAt = now + + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS; + } else if (!rosterProofUpgraded) { + // Pure growth preserves valid credits for retained identities, but the + // newly observed peer is uncredited and immediately breaks backoff. + record.phase = 'collecting'; + record.backoffKind = undefined; + record.nextRetryAt = 0; + record.collectionDeadlineAt = now + + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS; + } + } else { + record.curatorRosterConfirmed = curatorRosterConfirmed; + } + if (rosterProofUpgraded) { + // A peer response gathered while curator discovery was unconfirmed is + // useful transport evidence, not authoritative absence proof. Reprobe + // the complete now-authoritative roster even when that roster also grew. + record.phase = 'collecting'; + record.backoffKind = undefined; + record.nextRetryAt = 0; + record.attemptedPeerIds.clear(); + record.cleanAbsentPeerIds.clear(); + record.lastAttemptedPeerId = undefined; + record.collectionDeadlineAt = now + + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS; + } + if (record.phase === 'backoff') { + if (now < record.nextRetryAt) { + this.touchVmReconcileRotationRecord(slotKey, record); + return { slotKey, record, suppressed: true }; + } + // A deadline only opens a new collection cycle. It never earns another + // failure/backoff without fresh clean-absence evidence from every peer. + record.phase = 'collecting'; + record.backoffKind = undefined; + record.attemptedPeerIds.clear(); + record.cleanAbsentPeerIds.clear(); + record.collectionDeadlineAt = now + + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS; + record.nextRetryAt = 0; + } else if (now >= record.collectionDeadlineAt) { + // Expired partial evidence fails open and releases its cache slot. Return + // evidence-free for this pass so a repeatedly ineligible roster cannot + // refresh all collecting entries just before capacity admission runs. + this.vmReconcileRotationState.delete(slotKey); + return { slotKey, suppressed: false }; + } + + this.touchVmReconcileRotationRecord(slotKey, record); + return { slotKey, record, suppressed: false }; + } + + enterVmReconcileRotationBackoff( + this: DKGAgent, + slotKey: string, + record: VmReconcileRotationRecord, + kind: NonNullable = 'clean-absence', + ): void { + record.failures += 1; + const exponentialBackoff = Math.min( + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS, + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_BASE_MS + * 2 ** Math.max(0, record.failures - 1), + ); + const jitterSample = createHash('sha256') + .update(`${this.peerId}\0${slotKey}\0${record.fingerprint}\0${record.failures}`) + .digest() + .readUInt32BE(0) / 0x1_0000_0000; + const backoff = Math.min( + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS, + Math.max(1, Math.round(exponentialBackoff * (0.8 + jitterSample * 0.4))), + ); + record.phase = 'backoff'; + record.backoffKind = kind; + record.collectionDeadlineAt = 0; + record.nextRetryAt = this.vmReconcileRotationNow() + backoff; + } + + vmReconcileUncreditedCandidateOrder( + this: DKGAgent, + record: VmReconcileRotationRecord, + ): string[] { + const candidates = [...record.candidatePeerIds]; + if (candidates.length === 0) return candidates; + const lastIndex = record.lastAttemptedPeerId === undefined + ? -1 + : candidates.indexOf(record.lastAttemptedPeerId); + const start = lastIndex < 0 ? 0 : (lastIndex + 1) % candidates.length; + return [ + ...candidates.slice(start), + ...candidates.slice(0, start), + ].filter((peerId) => !record.attemptedPeerIds.has(peerId)); + } + + findVmReconcileRotationReplacement( + this: DKGAgent, + requestingCgId?: string, + ): [string, VmReconcileRotationRecord] | undefined { + if (this.vmReconcileRotationState.size < DKGAgentBase.VM_RECONCILE_CACHE_MAX_ENTRIES) { + return undefined; + } + const now = this.vmReconcileRotationNow(); + for (const entry of this.vmReconcileRotationState) { + const [, record] = entry; + if ( + (record.phase === 'backoff' && now < record.nextRetryAt) + || (record.phase === 'collecting' && now < record.collectionDeadlineAt) + ) continue; + return entry; + } + if (!requestingCgId) return undefined; + const countsByCg = new Map(); + for (const record of this.vmReconcileRotationState.values()) { + countsByCg.set(record.localCgId, (countsByCg.get(record.localCgId) ?? 0) + 1); + } + if ((countsByCg.get(requestingCgId) ?? 0) !== 0) return undefined; + for (const entry of this.vmReconcileRotationState) { + if ((countsByCg.get(entry[1].localCgId) ?? 0) > 1) return entry; + } + return undefined; + } + + canInstallVmReconcileRotationRecord(this: DKGAgent, requestingCgId?: string): boolean { + if (this.vmReconcileRotationState.size < DKGAgentBase.VM_RECONCILE_CACHE_MAX_ENTRIES) { + return true; + } + return this.findVmReconcileRotationReplacement(requestingCgId) !== undefined; + } + + installVmReconcileRotationRecord( + this: DKGAgent, + slotKey: string, + record: VmReconcileRotationRecord, + ): boolean { + if (this.vmReconcileRotationClosed || this.vmReconcileRotationState.has(slotKey)) { + return false; + } + const replacement = this.findVmReconcileRotationReplacement(record.localCgId); + if (!replacement) { + if (this.vmReconcileRotationState.size >= DKGAgentBase.VM_RECONCILE_CACHE_MAX_ENTRIES) { + return false; + } + this.vmReconcileRotationState.set(slotKey, record); + return this.vmReconcileRotationState.get(slotKey) === record; + } + + // Donation and requester installation are one synchronous state transition. + // Restore the donor if installation exits or throws before ownership moves. + const [replacementKey, replacementRecord] = replacement; + let installed = false; + this.vmReconcileRotationState.delete(replacementKey); + try { + if (this.vmReconcileRotationClosed || this.vmReconcileRotationState.has(slotKey)) { + return false; + } + this.vmReconcileRotationState.set(slotKey, record); + installed = this.vmReconcileRotationState.get(slotKey) === record; + return installed; + } finally { + if (!installed && !this.vmReconcileRotationState.has(replacementKey)) { + this.vmReconcileRotationState.set(replacementKey, replacementRecord); + } + } + } + + clearVmReconcileRotationStateForContextGraph( + this: DKGAgent, + localCgId: string, + ): void { + const prefix = `${localCgId}\0`; + for (const key of this.vmReconcileRotationState.keys()) { + if (key.startsWith(prefix)) this.vmReconcileRotationState.delete(key); + } + this.vmReconcileRotationAdmissionCursorByCg.delete(localCgId); + } + + closeVmReconcileRotationState(this: DKGAgent): void { + this.vmReconcileLifecycleController?.abort(); + this.vmReconcileLifecycleGeneration = (this.vmReconcileLifecycleGeneration ?? 0) + 1; + this.vmReconcileRotationClosed = true; + // Some lifecycle tests intentionally construct a narrow partial agent + // without running the base constructor. Shutdown must remain best-effort + // for that supported test seam and never mask later teardown failures. + this.vmReconcileRotationState?.clear(); + this.vmReconcileRotationAdmissionCursorByCg?.clear(); + this.vmReconcileCuratorPeersByCg?.clear(); + this.vmReconcileCuratorPageCursorByCg?.clear(); + } + + openVmReconcileRotationState(this: DKGAgent): void { + if (!this.vmReconcileLifecycleController || this.vmReconcileLifecycleController.signal.aborted) { + this.vmReconcileLifecycleController = new AbortController(); + } + this.vmReconcileRotationClosed = false; + } + + vmReconcileRecoveryTargetMatches( + this: DKGAgent, + expected: OrdinalRecoveryTarget, + actual: OrdinalRecoveryTarget, + ): boolean { + return expected.localCgId === actual.localCgId + && expected.onChainCgId === actual.onChainCgId + && expected.ordinal === actual.ordinal + && expected.ual === actual.ual + && expected.merkleRoot.toLowerCase() === actual.merkleRoot.toLowerCase(); + } + + settleVmReconcileRotationAttempt( + this: DKGAgent, + target: OrdinalRecoveryTarget, + peerId: string | undefined, + disposition: 'found' | 'clean-absent' | 'incomplete', + expectedCandidatePeerIds: readonly string[], + capturedRecord: VmReconcileRotationRecord, + unavailablePeerIds: ReadonlySet = new Set(), + ): void { + if (this.vmReconcileRotationClosed) return; + const slotKey = this.vmReconcileRotationSlotKey(target); + if (this.vmReconcileRotationState.get(slotKey) !== capturedRecord) return; + if (!this.vmReconcilePeerMembershipMatches( + capturedRecord.candidatePeerIds, + expectedCandidatePeerIds, + )) return; + if (peerId !== undefined && !capturedRecord.candidatePeerIds.has(peerId)) return; + + if (peerId !== undefined) { + capturedRecord.attemptedPeerIds.add(peerId); + if (disposition === 'clean-absent') capturedRecord.cleanAbsentPeerIds.add(peerId); + // Preserve fairly accumulated proof progress while other targets share + // the bounded peer budget. A cycle expires only after this slot itself + // stops making physical progress for the effective maximum. + capturedRecord.collectionDeadlineAt = this.vmReconcileRotationNow() + + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS; + } + const scheduledEveryPeer = [...capturedRecord.candidatePeerIds] + .every((candidatePeerId) => capturedRecord.attemptedPeerIds.has(candidatePeerId) + || unavailablePeerIds.has(candidatePeerId)); + const cleanAbsentFromEveryPeer = [...capturedRecord.candidatePeerIds] + .every((candidatePeerId) => capturedRecord.cleanAbsentPeerIds.has(candidatePeerId)); + if (cleanAbsentFromEveryPeer && capturedRecord.curatorRosterConfirmed) { + this.enterVmReconcileRotationBackoff(slotKey, capturedRecord, 'clean-absence'); + } else if (scheduledEveryPeer && capturedRecord.curatorRosterConfirmed) { + // This is retry suppression only, never absence proof. It prevents a + // legacy peer that ignores the exact filter from replaying the same + // bounded prefix at elevated priority every sweep. + this.enterVmReconcileRotationBackoff(slotKey, capturedRecord, 'incomplete-cycle'); + } + this.touchVmReconcileRotationRecord(slotKey, capturedRecord); + } + + creditVmReconcileCleanAbsence( + this: DKGAgent, + target: OrdinalRecoveryTarget, + peerId: string, + expectedCandidatePeerIds: readonly string[], + capturedRecord: VmReconcileRotationRecord, + ): void { + this.settleVmReconcileRotationAttempt( + target, + peerId, + 'clean-absent', + expectedCandidatePeerIds, + capturedRecord, + ); + } + shouldRunVmReconcileActiveFetch(this: DKGAgent, localCgId: string): boolean { const now = Date.now(); this.pruneVmReconcileState(now); @@ -3610,6 +4406,27 @@ export class SwmHostModeMethods extends DKGAgentBase { this.vmReconcileCatchupPeerOrder.delete(oldestKey); this.vmReconcileCatchupPeerCursor.delete(oldestKey); } + while (this.vmReconcileCuratorPeersByCg.size > DKGAgentBase.VM_RECONCILE_CG_STATE_MAX_ENTRIES) { + const oldestKey = this.vmReconcileCuratorPeersByCg.keys().next().value; + if (oldestKey === undefined) break; + this.vmReconcileCuratorPeersByCg.delete(oldestKey); + } + while ( + this.vmReconcileCuratorPageCursorByCg.size + > DKGAgentBase.VM_RECONCILE_CG_STATE_MAX_ENTRIES + ) { + const oldestKey = this.vmReconcileCuratorPageCursorByCg.keys().next().value; + if (oldestKey === undefined) break; + this.vmReconcileCuratorPageCursorByCg.delete(oldestKey); + } + while ( + this.vmReconcileRotationAdmissionCursorByCg.size + > DKGAgentBase.VM_RECONCILE_CG_STATE_MAX_ENTRIES + ) { + const oldestKey = this.vmReconcileRotationAdmissionCursorByCg.keys().next().value; + if (oldestKey === undefined) break; + this.vmReconcileRotationAdmissionCursorByCg.delete(oldestKey); + } } clearVmReconcileStateForContextGraph(this: DKGAgent, localCgId: string): void { @@ -3625,12 +4442,20 @@ export class SwmHostModeMethods extends DKGAgentBase { this.deleteVmReconcileNegativeCacheEntry(cacheKey); } } + for (const [cacheKey, hydratedLocalCgId] of this.vmReconcileNegativeCacheHydrated) { + if (hydratedLocalCgId === localCgId) { + this.vmReconcileNegativeCacheHydrated.delete(cacheKey); + } + } void this.config.contextGraphSubscriptionStore ?.deleteVmReconcileNegativesForContextGraph?.(localCgId) .catch(() => { // Best-effort durable cleanup; generation checks still reject stale rows. }); this.reconcileCursors.delete(localCgId); + this.clearVmReconcileRotationStateForContextGraph(localCgId); + this.vmReconcileCuratorPeersByCg.delete(localCgId); + this.vmReconcileCuratorPageCursorByCg.delete(localCgId); this.vmReconcileFetchCooldownAt.delete(localCgId); this.vmReconcileCatchupPeerCursor.delete(localCgId); this.vmReconcileCatchupPeerOrder.delete(localCgId); @@ -3689,7 +4514,13 @@ export class SwmHostModeMethods extends DKGAgentBase { targets: readonly OrdinalRecoveryTarget[], headBlock: number | undefined, isTargetCurrent: () => boolean, + signal?: AbortSignal, ): Promise { + const rotationGeneration = this.vmReconcileLifecycleGeneration; + const isRecoveryCurrent = () => !this.vmReconcileRotationClosed + && !signal?.aborted + && this.vmReconcileLifecycleGeneration === rotationGeneration + && isTargetCurrent(); const ctx = createOperationContext('system'); const noRecovery = ( continuationOrdinal?: number, @@ -3700,18 +4531,119 @@ export class SwmHostModeMethods extends DKGAgentBase { continuationOrdinal, cooldownOnly, }); - if (!isTargetCurrent() || targets.length === 0) return noRecovery(); + if (!isRecoveryCurrent() || targets.length === 0) return noRecovery(); + + const expectedOnChainCgId = onChainCgId.toString(); + const currentTargets = targets.filter((target) => + target.localCgId === localCgId && target.onChainCgId === expectedOnChainCgId); + if (currentTargets.length === 0) return noRecovery(); + const admissionCursor = ( + this.vmReconcileRotationAdmissionCursorByCg.get(localCgId) ?? 0 + ) % currentTargets.length; + const admissionDistance = (index: number) => ( + index - admissionCursor + currentTargets.length + ) % currentTargets.length; + + // Suppression consults only the already-observed, capped connection view. + // This is intentionally before curator resolution, dialing, protocol waits, + // and admission probes. Every target reached this method only after the + // production ordinal/finalization check proved it still pending locally. + const observedCandidatePeerIds = this.vmReconcileObservedCandidatePeerIds(localCgId); + const now = this.vmReconcileRotationNow(); + const initiallyOwnedSlotKeys = new Set(currentTargets.flatMap((target) => { + const slotKey = this.vmReconcileRotationSlotKey(target); + const record = this.vmReconcileRotationState.get(slotKey); + return record?.fingerprint === this.vmReconcileRotationFingerprint(target) + ? [slotKey] + : []; + })); + const hasUnownedTarget = currentTargets.some((target) => + !initiallyOwnedSlotKeys.has(this.vmReconcileRotationSlotKey(target))); + const reservedReplacementSlotKey = hasUnownedTarget + ? this.findVmReconcileRotationReplacement(localCgId)?.[0] + : undefined; + const initialPreparations = currentTargets + .map((target, index) => ({ + index, + target, + hasOwnedRecord: initiallyOwnedSlotKeys.has(this.vmReconcileRotationSlotKey(target)), + })) + // Use the same fair admission order before network work. Besides handing + // expired capacity to a waiter, this makes an all-live saturated cache + // return below without paying curator-resolution cost for work that + // cannot retain its retry state. + .sort((left, right) => Number(left.hasOwnedRecord) - Number(right.hasOwnedRecord) + || admissionDistance(left.index) - admissionDistance(right.index)) + .map(({ index, target }) => { + const slotKey = this.vmReconcileRotationSlotKey(target); + const existing = this.vmReconcileRotationState.get(slotKey); + if (!existing || existing.fingerprint !== this.vmReconcileRotationFingerprint(target)) { + // The pre-network pass only consults already-earned suppression. A new + // cycle is installed after curator resolution so its first roster is + // authoritative-first; a stale fingerprint is invalidated immediately. + if (existing) this.vmReconcileRotationState.delete(slotKey); + const capacityAvailable = this.canInstallVmReconcileRotationRecord(localCgId); + return { + index, + target, + prepared: { + slotKey, + record: undefined, + suppressed: this.vmReconcileRotationClosed || !capacityAvailable, + }, + }; + } + if (slotKey === reservedReplacementSlotKey) { + // Keep the donor intact, but do not renew it before the waiter reaches + // post-resolution installation. An earlier lifecycle exit leaves the + // original record untouched; a successful install replaces it atomically. + return { + index, + target, + prepared: { slotKey, record: existing, suppressed: true }, + }; + } + return { + index, + target, + prepared: this.prepareVmReconcileRotationTarget( + target, + observedCandidatePeerIds, + now, + existing.curatorRosterConfirmed, + ), + }; + }) + .sort((left, right) => left.index - right.index); + const initiallyEligible = initialPreparations + .filter(({ prepared }) => !prepared.suppressed) + .map(({ target }) => target); + if (initiallyEligible.length === 0) { + const suppressedRecords = initialPreparations + .map(({ prepared }) => prepared.record) + .filter((record): record is VmReconcileRotationRecord => record !== undefined); + const nextRetryInMs = suppressedRecords.length === 0 + ? 0 + : Math.max(0, Math.min(...suppressedRecords.map((record) => record.nextRetryAt)) - now); + this.log.info( + ctx, + `VM exact fetch for "${localCgId}" skipped by exact-recovery backoff ` + + `(slots=${suppressedRecords.length} candidates=${observedCandidatePeerIds.length} ` + + `failures=${Math.max(0, ...suppressedRecords.map((record) => record.failures))} ` + + `retryInMs=${Math.round(nextRetryInMs)})`, + ); + return noRecovery(); + } // Damping: the batched path deliberately skips the per-UAL negative cache // (consulting it primes connections to every discovered agent — the walk // this path exists to avoid), so the per-CG active-fetch cooldown is the - // only damper between this batch and the network. A wholly unproductive - // batch (e.g. the curator is offline) therefore costs one bounded exact - // fetch per sweep interval, not one per reconcile pass. Progress clears - // the cooldown below so a draining backlog proceeds slice after slice. + // short-term damper between fresh rotation attempts. Completed clean- + // absence rotations use the slot-specific exponential backoff above; + // transient/incomplete attempts retain this sweep-interval cooldown. if (!this.shouldRunVmReconcileActiveFetch(localCgId)) { this.log.info(ctx, `VM exact fetch for "${localCgId}" skipped by per-CG cooldown`); - return noRecovery(targets[0]?.ordinal, true); + return noRecovery(initiallyEligible[0]?.ordinal, true); } // Capture the authenticated join-approval hint before consulting metadata: @@ -3720,90 +4652,295 @@ export class SwmHostModeMethods extends DKGAgentBase { // registry resolver is authoritative for `0x…/slug` graphs and can return // every node registered to that curator wallet. const approvedCuratorPeerId = this.preferredSyncPeers.get(localCgId); - const curatorResolution = await this.resolveCuratorPeerIdsForCg(localCgId) - .catch(() => ({ peerIds: [] as string[], curatorIsLocal: false, legacyTripleResolved: false })); + const cachedCuratorPeerIds = [ + ...(this.vmReconcileCuratorPeersByCg.get(localCgId) ?? []), + ]; + const curatorPageCursor = this.vmReconcileCuratorPageCursorByCg.get(localCgId); + const curatorResolution = await this.resolveCuratorPeerIdsForCg(localCgId, { + maxPeerIds: DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX, + // Once overflow is proven, expose exactly one new ordered peer per pass. + // One target can spend only one peer attempt, so advancing by more would + // skip candidates when a CG has a single missing KA. + pagePeerIds: 1, + afterPeerId: curatorPageCursor, + signal, + isCurrent: isRecoveryCurrent, + }) + .catch(() => ({ + peerIds: [] as string[], + curatorIsLocal: false, + legacyTripleResolved: false, + lookupFailed: true, + overflowed: false, + nextPageAfterPeerId: undefined, + })); + if (!isRecoveryCurrent()) return noRecovery(); + const allResolvedCuratorPeerIds = [...new Set(curatorResolution.peerIds + .filter((peerId) => peerId && peerId !== this.peerId))] + .sort((left, right) => left.localeCompare(right)); + const curatorRosterOverflow = curatorResolution.overflowed === true + || allResolvedCuratorPeerIds.length > DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX; + if (curatorRosterOverflow) { + this.log.warn( + ctx, + `VM exact fetch curator roster for "${localCgId}" exceeds bounded proof capacity ` + + `(ordered transport page=${allResolvedCuratorPeerIds.length}, ` + + `proofCap=${DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX}); ` + + 'walking the registry without negative-proof suppression', + ); + } + const resolutionSucceeded = curatorResolution.lookupFailed !== true + && !curatorRosterOverflow; + // Even an invalid oversized result remains useful for bounded fail-open + // transport. Rotate a bounded window through it using the existing cache as + // the cursor: a fixed prefix (or a formerly authoritative cached roster) + // could otherwise hide a newly added holder forever. The window remains + // explicitly unconfirmed below, so it can never support absence proof. + const overflowTransportUniverse = [...new Set([ + ...allResolvedCuratorPeerIds, + approvedCuratorPeerId, + ].filter((peerId): peerId is string => Boolean(peerId && peerId !== this.peerId)))]; + const cachedOverflowStart = cachedCuratorPeerIds.length > 0 + ? overflowTransportUniverse.indexOf(cachedCuratorPeerIds[0]!) + : -1; + const overflowWindowStart = cachedOverflowStart < 0 + ? 0 + : (cachedOverflowStart + 1) % overflowTransportUniverse.length; + const overflowTransportPeerIds = Array.from( + { length: Math.min( + DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX, + overflowTransportUniverse.length, + ) }, + (_, offset) => overflowTransportUniverse[ + (overflowWindowStart + offset) % overflowTransportUniverse.length + ]!, + ); + const resolvedCuratorPeerIds = curatorRosterOverflow + ? curatorResolution.nextPageAfterPeerId + ? allResolvedCuratorPeerIds + : overflowTransportPeerIds + : allResolvedCuratorPeerIds; let legacyPreferredPeerId: string | undefined; - if (curatorResolution.peerIds.length === 0) { + if (resolutionSucceeded && !curatorResolution.curatorIsLocal + && resolvedCuratorPeerIds.length === 0) { legacyPreferredPeerId = await this.resolvePreferredSyncPeerId(localCgId); } + if (!isRecoveryCurrent()) return noRecovery(); + const authoritativeCuratorPeerIds = resolutionSucceeded + ? resolvedCuratorPeerIds + : curatorRosterOverflow + ? resolvedCuratorPeerIds + : cachedCuratorPeerIds; const curatorPeerIds = [...new Set([ - approvedCuratorPeerId, - ...curatorResolution.peerIds, + ...authoritativeCuratorPeerIds, legacyPreferredPeerId, + approvedCuratorPeerId, ].filter((peerId): peerId is string => Boolean(peerId && peerId !== this.peerId)))] - .slice(0, 3); - for (const peerId of curatorPeerIds) { - await this.ensurePeerConnected(peerId).catch((error) => { - this.log.info( - ctx, - `VM exact fetch could not connect curator peer ${peerId.slice(-8)}: ${error instanceof Error ? error.message : String(error)}`, - ); - }); + .slice(0, DKGAgentBase.VM_RECONCILE_EXACT_ROSTER_MAX); + // Persist the bounded full authoritative roster. Individual passes still + // connect/probe at most VM_RECONCILE_EXACT_PEER_MAX peers, while each + // target's rotation record carries progress across those windows. + if (resolutionSucceeded) { + this.vmReconcileCuratorPeersByCg.delete(localCgId); + this.vmReconcileCuratorPageCursorByCg.delete(localCgId); + } else if (curatorResolution.nextPageAfterPeerId) { + this.vmReconcileCuratorPageCursorByCg.delete(localCgId); + this.vmReconcileCuratorPageCursorByCg.set( + localCgId, + curatorResolution.nextPageAfterPeerId, + ); + } + if (!curatorResolution.curatorIsLocal && curatorPeerIds.length > 0) { + this.vmReconcileCuratorPeersByCg.delete(localCgId); + this.vmReconcileCuratorPeersByCg.set(localCgId, curatorPeerIds); } + this.pruneVmReconcileState(); const connectedByPeerId = new Map( this.node.libp2p.getConnections() .map((connection) => [connection.remotePeer.toString(), connection.remotePeer]), ); - const connected = [...connectedByPeerId.values()]; - const connectedPeerIds = new Set(connectedByPeerId.keys()); - const orderedConnectedPeerIds = this.selectCatchupPeers( - connected, - approvedCuratorPeerId ?? curatorPeerIds[0], - false, - ).map((peer) => peer.toString()); - const orderedPeerIds = [...new Set([ - ...curatorPeerIds.filter((peerId) => connectedPeerIds.has(peerId)), - ...orderedConnectedPeerIds, - ])].slice(0, 3); - const rotationCandidates = orderedPeerIds - .map((peerId) => connectedByPeerId.get(peerId)) - .filter((peer): peer is NonNullable => peer !== undefined); - const peerPriorityRanks = new Map( - orderedPeerIds.map((peerId) => [ - peerId, - curatorPeerIds.includes(peerId) ? 2 : this.knownCorePeerIds.has(peerId) ? 1 : 0, - ]), - ); - // Preserve curator preference for the first attempt, then rotate the full - // connected order one peer per network-eligible window. Exact recovery - // often has a single pending KA, so replaying the static preferred-first - // order would otherwise pin that KA to the same unproductive peer forever. - const rotationAnchor = this.selectCatchupPeerWindow(rotationCandidates, { - maxPeers: 1, - peerRotationKey: localCgId, - peerPriorityRanks, - })[0]?.toString(); - const rotationStart = rotationAnchor === undefined - ? 0 - : Math.max(0, orderedPeerIds.indexOf(rotationAnchor)); - const peerIds = [ - ...orderedPeerIds.slice(rotationStart), - ...orderedPeerIds.slice(0, rotationStart), - ]; + // Use the exact same memory-only canonicalizer as the suppression gate. + // Curator resolution may connect a missing peer, but it must not substitute + // a different ranking algorithm and invalidate an otherwise stable roster. + const orderedPeerIds = this.vmReconcileObservedCandidatePeerIds(localCgId); + + // Curator preparation may have grown or shrunk the connected candidate + // set. Re-evaluate every target against that observed change. Any roster + // change breaks backoff and starts a fresh proof cycle. + const preparedEntries = currentTargets + .map((target, index) => ({ + index, + target, + hasOwnedRecord: initiallyOwnedSlotKeys.has(this.vmReconcileRotationSlotKey(target)), + })) + // At a full cap, give deferred targets first claim on an expired slot. + // Otherwise an expired owner encountered first would renew itself before + // any waiter could enter, starving stable-order overflow indefinitely. + .sort((left, right) => Number(left.hasOwnedRecord) - Number(right.hasOwnedRecord) + || admissionDistance(left.index) - admissionDistance(right.index)) + .map(({ index, target }) => ({ + index, + target, + prepared: this.prepareVmReconcileRotationTarget( + target, + orderedPeerIds, + this.vmReconcileRotationNow(), + resolutionSucceeded, + ), + })) + .sort((left, right) => left.index - right.index); + const newlyAdmitted = preparedEntries + .filter(({ target, prepared }) => prepared.record + && !initiallyOwnedSlotKeys.has(this.vmReconcileRotationSlotKey(target))); + if (newlyAdmitted.length > 0) { + const lastAdmitted = newlyAdmitted.reduce((latest, entry) => ( + admissionDistance(entry.index) > admissionDistance(latest.index) ? entry : latest + )); + this.vmReconcileRotationAdmissionCursorByCg.delete(localCgId); + this.vmReconcileRotationAdmissionCursorByCg.set( + localCgId, + (lastAdmitted.index + 1) % currentTargets.length, + ); + } + const eligible = preparedEntries + .map((entry) => { + const { record } = entry.prepared; + if ( + record + && this.vmReconcileRotationState.get(entry.prepared.slotKey) !== record + ) { + // A later slot may have replaced an expired record while the batch + // was prepared. Defer the now-unowned target: elevated transport + // without retained retry state would violate the pressure bound. + return { + ...entry, + prepared: { slotKey: entry.prepared.slotKey, suppressed: true }, + }; + } + return entry; + }) + .filter((entry) => !entry.prepared.suppressed) + // Installed collecting records get first use of the bounded peer set so + // overflow cannot consume the one peer they still need to complete. The + // original target order remains stable within each class. + .sort((left, right) => { + const leftInstalled = left.prepared.record + && this.vmReconcileRotationState.get(left.prepared.slotKey) === left.prepared.record + ? 1 : 0; + const rightInstalled = right.prepared.record + && this.vmReconcileRotationState.get(right.prepared.slotKey) === right.prepared.record + ? 1 : 0; + return rightInstalled - leftInstalled || left.index - right.index; + }); + const outcomes = new Map(); const attemptedOrdinals = new Set(); - let remaining = [...targets]; - let attemptedFetch = false; - - for (const peerId of peerIds) { - if (!isTargetCurrent() || remaining.length === 0) break; - const connectedPeer = connectedByPeerId.get(peerId); - if (!connectedPeer || !(await this.waitForSyncProtocol(connectedPeer))) continue; - // Network boundary: a merely-connected peer is not necessarily admitted - // to this DKG network (curator hints and getConnections() both predate - // the identity probe). Never send an authenticated exact request to an - // unverified or rejected peer. - if (!(await this.ensurePeerAdmittedForRecovery(peerId, ctx, 'VM exact fetch'))) continue; + const usedPeerIds = new Set(); + const consideredPeerIds = new Set(); + const unavailablePeerIds = new Set(); + let recoveryWorkRan = false; + + for (const entry of eligible) { + if (!isRecoveryCurrent()) break; + const { target } = entry; + const record = entry.prepared.record; + const installedRecord = record + && this.vmReconcileRotationState.get(this.vmReconcileRotationSlotKey(target)) === record + ? record + : undefined; + const candidatePeerIds = installedRecord + ? [...installedRecord.candidatePeerIds] + : orderedPeerIds; + + // Rotate every physical outcome, including incomplete responses, without + // conflating the attempt cursor with clean-absence evidence. Try the + // target's next available peer. Protocol/admission failures remain + // uncredited, but consume this target's turn so another missing KA gets + // the next physical peer slot in the same bounded pass. + let peerId: string | undefined; + const candidateOrder = installedRecord + ? this.vmReconcileUncreditedCandidateOrder(installedRecord) + : orderedPeerIds; + for (const candidatePeerId of candidateOrder) { + if (usedPeerIds.has(candidatePeerId) || unavailablePeerIds.has(candidatePeerId)) continue; + if ( + !consideredPeerIds.has(candidatePeerId) + && consideredPeerIds.size >= DKGAgentBase.VM_RECONCILE_EXACT_PEER_MAX + ) break; + consideredPeerIds.add(candidatePeerId); + attemptedOrdinals.add(target.ordinal); + if (installedRecord) { + installedRecord.lastAttemptedPeerId = candidatePeerId; + installedRecord.attemptedPeerIds.add(candidatePeerId); + installedRecord.collectionDeadlineAt = this.vmReconcileRotationNow() + + DKGAgentBase.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS; + this.touchVmReconcileRotationRecord( + this.vmReconcileRotationSlotKey(target), + installedRecord, + ); + } + let connectedPeer = connectedByPeerId.get(candidatePeerId); + if (!connectedPeer) { + await this.ensurePeerConnected(candidatePeerId, { signal }).catch((error) => { + this.log.info( + ctx, + `VM exact fetch could not connect candidate peer ${candidatePeerId.slice(-8)}: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + if (!isRecoveryCurrent()) return noRecovery(); + const connection = this.node.libp2p.getConnections() + .find((candidate) => candidate.remotePeer.toString() === candidatePeerId); + connectedPeer = connection?.remotePeer; + if (connectedPeer) connectedByPeerId.set(candidatePeerId, connectedPeer); + } + recoveryWorkRan = true; + const protocolReady = connectedPeer + ? await this.waitForSyncProtocol(connectedPeer, signal) + : false; + if (!isRecoveryCurrent()) return noRecovery(); + if (!connectedPeer || !protocolReady) { + unavailablePeerIds.add(candidatePeerId); + break; + } + // Network boundary: a merely-connected peer is not necessarily + // admitted to this DKG network. Never send an authenticated exact + // request to an unverified or rejected peer. + const peerAdmitted = await this.ensurePeerAdmittedForRecovery( + candidatePeerId, + ctx, + 'VM exact fetch', + signal, + ); + if (!isRecoveryCurrent()) return noRecovery(); + if (!peerAdmitted) { + unavailablePeerIds.add(candidatePeerId); + break; + } + peerId = candidatePeerId; + break; + } + if (!peerId) { + if (installedRecord) { + this.settleVmReconcileRotationAttempt( + target, + undefined, + 'incomplete', + candidatePeerIds, + installedRecord, + unavailablePeerIds, + ); + } + continue; + } + usedPeerIds.add(peerId); // A recovery target can approach the frame budget by itself. Each peer // attempt therefore consumes at most one queue item, and each queue item // consumes at most one peer attempt per eligible pass. A still-pending // item rotates behind untouched work so one unavailable KA cannot spend // every peer budget and starve the rest of the queue. - const [target, ...deferredTargets] = remaining; - if (!target) break; - if (attemptedOrdinals.has(target.ordinal)) break; + if (!isRecoveryCurrent()) return noRecovery(); this.emitReplication({ contextGraphId: localCgId, onChainCgId: onChainCgId.toString(), @@ -3814,17 +4951,26 @@ export class SwmHostModeMethods extends DKGAgentBase { detail: 'exact-asset', }); + let disposition: 'found' | 'clean-absent' | 'incomplete' = 'incomplete'; try { - attemptedFetch = true; - attemptedOrdinals.add(target.ordinal); - const result = await this.syncExactKnowledgeAssetsFromPeer( + if (installedRecord) { + installedRecord.lastAttemptedPeerId = peerId; + this.touchVmReconcileRotationRecord( + this.vmReconcileRotationSlotKey(target), + installedRecord, + ); + } + const detailed = await this.syncExactKnowledgeAssetsFromPeerDetailed( peerId, localCgId, [target.ual], + { signal, isCurrent: isRecoveryCurrent }, ); + const { result } = detailed; + disposition = detailed.disposition; this.log.info( ctx, - `VM exact fetch for "${localCgId}" from ${peerId.slice(-8)}: requested=1 fetched=${result.fetchedDataTriples + result.fetchedMetaTriples} inserted=${result.insertedTriples} failed=${result.failedPeers + result.failedPhases} deferred=${result.deferredBackpressure}`, + `VM exact fetch for "${localCgId}" from ${peerId.slice(-8)}: requested=1 fetched=${result.fetchedDataTriples + result.fetchedMetaTriples} inserted=${result.insertedTriples} failed=${result.failedPeers + result.failedPhases} deferred=${result.deferredBackpressure} disposition=${disposition}`, ); } catch (error) { this.log.info( @@ -3833,31 +4979,70 @@ export class SwmHostModeMethods extends DKGAgentBase { ); } - if (!isTargetCurrent()) break; + // The exact request may have completed after unsubscribe, rebind, or + // shutdown. Its authenticated materialization is already fail-closed, + // but no process-local evidence or cooldown may outlive that lifecycle. + if (!isRecoveryCurrent()) return noRecovery(); const outcome = await this.reconcileChainOrdinal( localCgId, onChainCgId, target.ordinal, headBlock, - { isTargetCurrent, deferActiveFetch: true }, + { isTargetCurrent: isRecoveryCurrent, deferActiveFetch: true }, ); + if (!isRecoveryCurrent()) return noRecovery(); outcomes.set(target.ordinal, outcome); if (outcome.status === 'pending' && outcome.recovery) { - remaining = [...deferredTargets, outcome.recovery]; + if (!installedRecord) continue; + if (this.vmReconcileRotationClosed) continue; + if ( + this.vmReconcileRotationState.get(this.vmReconcileRotationSlotKey(target)) + !== installedRecord + ) continue; + const candidateMembershipAfter = this.vmReconcileObservedCandidatePeerIds( + localCgId, + ); + if (!this.vmReconcilePeerMembershipMatches( + installedRecord.candidatePeerIds, + candidateMembershipAfter, + )) { + // Membership changed while the request was in flight. Invalidate the + // cycle before considering its response, then fail open. + this.prepareVmReconcileRotationTarget( + outcome.recovery, + candidateMembershipAfter, + this.vmReconcileRotationNow(), + ); + } else if (this.vmReconcileRecoveryTargetMatches(target, outcome.recovery)) { + this.settleVmReconcileRotationAttempt( + target, + peerId, + disposition, + candidatePeerIds, + installedRecord, + unavailablePeerIds, + ); + } } else { - remaining = deferredTargets; + this.vmReconcileRotationState.delete(this.vmReconcileRotationSlotKey(target)); } } - // Mirror the inline path's cooldown policy: an unreachable network must - // not consume the fetch budget, and completed work resets the damper so - // the trailing `hasMore` slice of a draining backlog fetches immediately. - // Only a batch that reached a peer and recovered nothing leaves the - // cooldown standing. + // Mirror the inline path's cooldown policy: a pass that performs no + // protocol/admission/transport work must not consume the fetch budget, and + // completed work resets the damper so the trailing `hasMore` slice of a + // draining backlog fetches immediately. + // Only a batch that reached a peer and recovered nothing retains the + // cooldown. Re-anchor that short damper at completion: a slow or legacy + // response may already have outlived the timestamp taken before transfer. + if (!isRecoveryCurrent()) return noRecovery(); const recoveredAny = [...outcomes.values()] .some((outcome) => outcome.status === 'reconciled' || outcome.status === 'already'); - if (!attemptedFetch || recoveredAny) { + if (!recoveryWorkRan || recoveredAny) { this.vmReconcileFetchCooldownAt.delete(localCgId); + } else { + this.vmReconcileFetchCooldownAt.delete(localCgId); + this.vmReconcileFetchCooldownAt.set(localCgId, Date.now()); } return { outcomes, @@ -3866,8 +5051,9 @@ export class SwmHostModeMethods extends DKGAgentBase { // attempts are rotated inside `remaining` to give untouched targets the // next peer, but once every submitted target has consumed one attempt // the outer fair scan must wrap from its watermark on the next cycle. - continuationOrdinal: targets.find( - (target) => !attemptedOrdinals.has(target.ordinal), + continuationOrdinal: currentTargets.find( + (target) => eligible.some((entry) => entry.target.ordinal === target.ordinal) + && !attemptedOrdinals.has(target.ordinal), )?.ordinal, cooldownOnly: false, }; @@ -3914,7 +5100,10 @@ export class SwmHostModeMethods extends DKGAgentBase { // Recently reconciled (live-burst guard): treat as already-done so the // cursor advances without redoing chain reads + an SWM scan. - if (this.recentReconciledUals.has(cacheKey)) return { status: 'already', blockNumber: versionBlock }; + if (this.recentReconciledUals.has(cacheKey)) { + this.clearVmReconcileRotationStateForSlot(localCgId, onChainCgId, ordinal); + return { status: 'already', blockNumber: versionBlock }; + } if (!options.deferActiveFetch && await this.shouldDeferVmReconcileByNegativeCache(cacheKey, localCgId)) { this.emitReplication({ @@ -3970,8 +5159,14 @@ export class SwmHostModeMethods extends DKGAgentBase { return { status: 'pending', recovery: { + localCgId, + onChainCgId: onChainCgId.toString(), ordinal, ual, + merkleRoot: Array.from( + merkleRoot, + (byte) => byte.toString(16).padStart(2, '0'), + ).join(''), kaId: kaId.toString(), reason: outcome, }, @@ -4061,6 +5256,7 @@ export class SwmHostModeMethods extends DKGAgentBase { switch (outcome) { case 'promoted': + this.clearVmReconcileRotationStateForSlot(localCgId, onChainCgId, ordinal); this.pruneVmReconcileCacheKeySiblings(cacheKey); this.deleteVmReconcileNegativeCacheEntry(cacheKey); this.recentReconciledUals.add(cacheKey); @@ -4070,6 +5266,7 @@ export class SwmHostModeMethods extends DKGAgentBase { }); return { status: 'reconciled', blockNumber: versionBlock }; case 'already-confirmed': + this.clearVmReconcileRotationStateForSlot(localCgId, onChainCgId, ordinal); this.pruneVmReconcileCacheKeySiblings(cacheKey); this.recentReconciledUals.add(cacheKey); this.emitReplication({ @@ -4079,6 +5276,7 @@ export class SwmHostModeMethods extends DKGAgentBase { this.deleteVmReconcileNegativeCacheEntry(cacheKey); return { status: 'already', blockNumber: versionBlock }; case 'stale-target': + this.clearVmReconcileRotationStateForSlot(localCgId, onChainCgId, ordinal); // A newer root won; do not prune its cache/recent state. this.recentReconciledUals.add(cacheKey); this.emitReplication({ diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 4893d23842..36d4de0f7c 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -480,6 +480,11 @@ export class SwmSubstrateMethods extends DKGAgentBase { const existing = this.subscribedContextGraphs.get(contextGraphId); if (!existing) return; + // A host-only Core may continue chain reconciliation after member + // unsubscribe, but peer-rotation evidence collected under the member + // lifecycle must not survive that ownership transition. + this.clearVmReconcileRotationStateForContextGraph(contextGraphId); + // Drop from the active sync scope so background sweeps no longer treat // this as a subscribed CG to keep current. const syncSet = new Set(this.config.syncContextGraphs ?? []); diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index e99152f1d6..39bc7c053a 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -753,6 +753,33 @@ export interface VmReconcileNegativeRecord { peerTopologyKey: string; } +/** Process-local evidence for one chain-ordinal exact-recovery rotation. */ +export interface VmReconcileRotationRecord { + localCgId: string; + onChainCgId: string; + ordinal: number; + fingerprint: string; + phase: 'collecting' | 'backoff'; + /** Retry suppression is distinct from authenticated clean-absence proof. */ + backoffKind?: 'clean-absence' | 'incomplete-cycle'; + candidatePeerIds: Set; + /** Peers physically attempted during the current proof cycle. */ + attemptedPeerIds: Set; + cleanAbsentPeerIds: Set; + /** + * A process-local curator lookup completed (or its bounded cached roster was + * reused). This is not cryptographic or network-wide completeness evidence; + * observed roster changes invalidate the cycle and backoff is time-bounded. + */ + curatorRosterConfirmed: boolean; + /** Monotonic bound after which a partial clean-absence proof restarts. */ + collectionDeadlineAt: number; + /** Cursor only; every physical attempt advances it, regardless of outcome. */ + lastAttemptedPeerId?: string; + failures: number; + nextRetryAt: number; +} + export interface ContextGraphSubscriptionRehydrationStatus { /** Non-system persisted rows governed by the rehydration cap. */ persistedTotal: number; diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 85f8ff7c3f..498f079ecb 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -396,6 +396,8 @@ import { deserializePendingSenderKeyEntry, } from './dkg-agent-swm-state.js'; import { DKGAgentBase, createListContextGraphsCacheInvalidatingStore } from './dkg-agent-base.js'; +import { VmReconcileShutdownTimeoutError } from './vm-reconcile-service.js'; +import { ContextGraphMembershipPersistShutdownTimeoutError } from './context-graph-membership-persist-scheduler.js'; import { reconcileAndAllocateKaNumber } from './allocator.js'; import { applyMixins } from './dkg-agent-apply-mixins.js'; import { OwnershipMethods } from './dkg-agent-ownership.js'; @@ -1630,14 +1632,22 @@ export class DKGAgent extends DKGAgentBase { async stop(): Promise { if (!this.started) return; - if (this.chainPoller) { - // Await so any in-flight poll (and its HTTP keep-alive socket) settles - // BEFORE we tear down the chain adapter — otherwise the RPC connection - // closure surfaces as an `ECONNRESET` unhandled rejection from inside - // ethers (the same flake that has been hitting `publisher [2/4]` in CI). - await this.chainPoller.stop(); - this.chainPoller = null; - } + // Fence membership persistence before any network callback can enqueue + // more work; the physical drain below completes before store teardown. + const membershipPersistDrain = this.contextGraphMembershipPersistence?.closeAndDrain() + ?? Promise.resolve(); + // Invalidate VM reconcile callbacks before waiting for the chain poller. + // A poll can be inside the KACG nudge's self-prime lookup; aborting the + // lifecycle first lets that lookup's bounded race release poller shutdown. + this.vmReconcileRuntimeReady = false; + this.graphScopedStoreClosed = true; + this.closeVmReconcileRotationState(); + const chainPoller = this.chainPoller; + // stop() fences new poll admission synchronously, but its in-flight poll + // joins the bounded physical-retirement drain below. An adapter that + // ignores cancellation must quarantine shutdown instead of preventing the + // retirement timeout from ever being reached. + const chainPollerDrain = chainPoller?.stop(); if (this.swmCleanupTimer) { clearInterval(this.swmCleanupTimer); this.swmCleanupTimer = null; @@ -1676,29 +1686,98 @@ export class DKGAgent extends DKGAgentBase { } // Close admission before any network/store teardown. Pending reconciles // are rejected immediately and therefore can never start after shutdown - // begins; an already-active pass gets a bounded grace period because - // cancelling midway could strand a partially applied VM transition. + // begins. Active callers receive a bounded grace period; generation and + // target fences prevent any late continuation from committing lifecycle + // state after the cancellation signal. + // Exact-absence rotations were cleared before stopping the chain poller, + // so a late in-flight response cannot restore process-local suppression. const vmReconcileDispatcher = this.vmReconcileDispatcher; - if (vmReconcileDispatcher) { - const drain = vmReconcileDispatcher.close(); - let drainTimedOut = false; - let timeoutHandle: ReturnType | undefined; - const timeout = new Promise((resolve) => { - timeoutHandle = setTimeout(() => { - drainTimedOut = true; - resolve(); - }, DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS); - timeoutHandle.unref?.(); - }); - await Promise.race([drain, timeout]); - if (timeoutHandle) clearTimeout(timeoutHandle); - if (drainTimedOut) { - this.log.warn( - createOperationContext('system'), - `DKGAgent.stop: ${vmReconcileDispatcher.snapshot().active} VM reconcile job(s) still active after ${DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS}ms drain bound — proceeding with shutdown`, - ); + const vmReconcileSweep = this.vmReconcileSweepInFlight; + const priorRetirement = this.vmReconcileRetirement; + // close() fences admission synchronously before the physical-set drain is + // sampled, so no dispatcher worker can appear behind an observed empty set. + const dispatcherDrain = vmReconcileDispatcher?.close(); + const drainPhysicalRuns = async (): Promise => { + while ( + (this.vmReconcilePhysicalRuns?.size ?? 0) > 0 + || (this.graphScopedStorePhysicalRuns?.size ?? 0) > 0 + ) { + const snapshot = [ + ...(this.vmReconcilePhysicalRuns ?? []), + ...(this.graphScopedStorePhysicalRuns ?? []), + ]; + await Promise.allSettled(snapshot); + for (const settled of snapshot) this.vmReconcilePhysicalRuns?.delete(settled); + for (const settled of snapshot) this.graphScopedStorePhysicalRuns?.delete(settled); + } + }; + const drains: Promise[] = [drainPhysicalRuns()]; + if (chainPollerDrain) drains.push(chainPollerDrain); + if (priorRetirement) drains.push(priorRetirement.catch(() => undefined)); + if (dispatcherDrain) drains.push(dispatcherDrain); + if (vmReconcileSweep) drains.push(vmReconcileSweep.catch(() => undefined)); + + let retirement!: Promise; + retirement = Promise.allSettled(drains).then(() => { + if (this.vmReconcileDispatcher === vmReconcileDispatcher) { + this.vmReconcileDispatcher = undefined; + } + if (this.vmReconcileSweepInFlight === vmReconcileSweep) { + this.vmReconcileSweepInFlight = null; + } + if (this.chainPoller === chainPoller) { + this.chainPoller = null; + } + if (this.vmReconcileRetirement === retirement) { + this.vmReconcileRetirement = null; } + }); + this.vmReconcileRetirement = retirement; + + let drainTimedOut = false; + let timeoutHandle: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timeoutHandle = setTimeout(() => { + drainTimedOut = true; + resolve(); + }, DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS); + timeoutHandle.unref?.(); + }); + await Promise.race([retirement, timeout]); + if (timeoutHandle) clearTimeout(timeoutHandle); + if (drainTimedOut) { + this.vmReconcileShutdownBlocked = true; + this.log.warn( + createOperationContext('system'), + `DKGAgent.stop: graph-scoped sync/reconciliation work did not physically retire within ${DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS}ms; store/network teardown is blocked until stop() is retried`, + ); + throw new VmReconcileShutdownTimeoutError(DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS); + } + this.vmReconcileShutdownBlocked = false; + let membershipDrainTimedOut = false; + let membershipTimeoutHandle: ReturnType | undefined; + const membershipTimeout = new Promise((resolve) => { + membershipTimeoutHandle = setTimeout(() => { + membershipDrainTimedOut = true; + resolve(); + }, DKGAgentBase.CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS); + membershipTimeoutHandle.unref?.(); + }); + await Promise.race([membershipPersistDrain, membershipTimeout]); + if (membershipTimeoutHandle) clearTimeout(membershipTimeoutHandle); + if (membershipDrainTimedOut) { + this.contextGraphMembershipPersistenceShutdownBlocked = true; + this.log.warn( + createOperationContext('system'), + `DKGAgent.stop: context-graph membership persistence did not drain within ` + + `${DKGAgentBase.CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS}ms; ` + + `store teardown is blocked until stop() is retried`, + ); + throw new ContextGraphMembershipPersistShutdownTimeoutError( + DKGAgentBase.CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS, + ); } + this.contextGraphMembershipPersistenceShutdownBlocked = false; this.coreHostRecordingsClosed = true; await this.drainCoreHostRecordings(); if (this.messengerOutboxTimer) { diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 52b73ad5a5..04bab01885 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -78,11 +78,19 @@ export { ContextGraphOnChainIdUnresolvedError, VmReconcileQueueClosedError, VmReconcileQueueFullError, + VmReconcileShutdownTimeoutError, + VM_RECONCILE_SHUTDOWN_TIMEOUT_ERROR_CODE, VmReconcileUnavailableError, type ContextGraphReconcileResult, type ContextGraphReconcileStatus, type VmReconcileSource, } from './vm-reconcile-service.js'; +export { + ContextGraphMembershipPersistQueueClosedError, + ContextGraphMembershipPersistQueueFullError, + ContextGraphMembershipPersistShutdownTimeoutError, + CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_ERROR_CODE, +} from './context-graph-membership-persist-scheduler.js'; export { buildEndorsementQuads, DKG_ENDORSES, DKG_ENDORSED_AT } from './endorse.js'; export { CclEvaluator, diff --git a/packages/agent/src/p2p/peer-connect.ts b/packages/agent/src/p2p/peer-connect.ts index a1ba68a93d..a5ff36b0c4 100644 --- a/packages/agent/src/p2p/peer-connect.ts +++ b/packages/agent/src/p2p/peer-connect.ts @@ -5,7 +5,7 @@ import { interface Libp2pLike { getConnections(): Array<{ remotePeer: { toString(): string } }>; - dial(peer: unknown): Promise; + dial(peer: unknown, options?: { signal?: AbortSignal }): Promise; peerStore: { merge(peer: unknown, update: { multiaddrs: unknown[] }): Promise; }; @@ -76,7 +76,11 @@ export async function ensurePeerConnected( libp2p: Libp2pLike, discovery: DiscoveryClient, peerId: string, + options: { signal?: AbortSignal } = {}, ): Promise { + if (options.signal?.aborted) { + throw new DOMException('Peer connection aborted', 'AbortError'); + } const existingConnections = libp2p.getConnections() .filter((conn) => conn.remotePeer.toString() === peerId); if (existingConnections.length > 0) { @@ -88,18 +92,25 @@ export async function ensurePeerConnected( const pid = peerIdFromString(peerId); try { - await libp2p.dial(pid); + await libp2p.dial(pid, { signal: options.signal }); return; } catch { - const agent = await discovery.findAgentByPeerId(peerId); + if (options.signal?.aborted) { + throw new DOMException('Peer connection aborted', 'AbortError'); + } + const agent = await discovery.findAgentByPeerId(peerId, { signal: options.signal }); + if (options.signal?.aborted) { + throw new DOMException('Peer connection aborted', 'AbortError'); + } if (!agent?.relayAddress) return; const { multiaddr } = await import('@multiformats/multiaddr'); const circuitAddr = multiaddr(`${agent.relayAddress}/p2p-circuit/p2p/${peerId}`); await libp2p.peerStore.merge(pid, { multiaddrs: [circuitAddr] }); - await libp2p.dial(pid); + await libp2p.dial(pid, { signal: options.signal }); } - } catch { + } catch (error) { + if (options.signal?.aborted) throw error; // Non-fatal — peer may be unreachable. } } diff --git a/packages/agent/src/p2p/protocol-readiness.ts b/packages/agent/src/p2p/protocol-readiness.ts index 223540543b..efd78c056c 100644 --- a/packages/agent/src/p2p/protocol-readiness.ts +++ b/packages/agent/src/p2p/protocol-readiness.ts @@ -4,8 +4,12 @@ export async function waitForPeerProtocol( protocol: string, attempts: number, delayMs: number, + signal?: AbortSignal, ): Promise { for (let attempt = 0; attempt < attempts; attempt++) { + if (signal?.aborted) { + throw new DOMException('Protocol readiness wait aborted', 'AbortError'); + } try { const peerInfo = await peerStore.get(peer as any); if (peerInfo.protocols.includes(protocol)) { @@ -16,7 +20,23 @@ export async function waitForPeerProtocol( } if (attempt < attempts - 1) { - await new Promise((resolve) => setTimeout(resolve, delayMs)); + await new Promise((resolve, reject) => { + let timer: ReturnType | undefined; + const cleanup = () => { + if (timer) clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + cleanup(); + reject(new DOMException('Protocol readiness wait aborted', 'AbortError')); + }; + timer = setTimeout(() => { + cleanup(); + resolve(); + }, delayMs); + if (signal?.aborted) onAbort(); + else signal?.addEventListener('abort', onAbort, { once: true }); + }); } } diff --git a/packages/agent/src/sync/auth/request-build.ts b/packages/agent/src/sync/auth/request-build.ts index ee92274f3d..e6b580056a 100644 --- a/packages/agent/src/sync/auth/request-build.ts +++ b/packages/agent/src/sync/auth/request-build.ts @@ -62,7 +62,8 @@ export interface SyncRequestEnvelope { * Additive, UNSIGNED exact-KA response filter. It can only narrow an already * authorized Context Graph read. Upgraded responders serve metadata and data * for these UALs only; old responders ignore it, while the upgraded requester - * still filters their full response before verification/storage. + * accepts and filters only a bounded legacy prefix that covers every requested + * descriptor. An over-limit or incomplete legacy response remains fail-closed. */ assetUals?: string[]; /** diff --git a/packages/agent/src/sync/backpressure.ts b/packages/agent/src/sync/backpressure.ts index 7b4eb8d7aa..6766a42247 100644 --- a/packages/agent/src/sync/backpressure.ts +++ b/packages/agent/src/sync/backpressure.ts @@ -1,3 +1,4 @@ +import { performance } from 'node:perf_hooks'; import { getMetrics, type OperationContext } from '@origintrail-official/dkg-core'; import { normalizeSyncAdmissionSource, @@ -62,7 +63,7 @@ function syncOperationClass(label: string): string { * admission source is what lets an operator attribute a saturated `sync-global` * queue to explicit catch-up versus sync-on-connect versus reconcile, and read * per-trigger queue/active ages straight off the snapshot. Both halves are - * closed sets (5 × 7), so the label space stays bounded and free of Context + * closed sets (5 × 8), so the label space stays bounded and free of Context * Graph and peer identifiers. */ function syncAdmissionOperation(payload: GlobalQueuePayload): string { @@ -73,6 +74,7 @@ let inflight = 0; let lastLimit: number | null = null; let lastQueueLimit: number | null = null; const queue = new PriorityAdmissionQueue({ + now: () => performance.now(), canRun: (entry) => inflight < entry.payload.limit, onStart: (entry) => { inflight += 1; @@ -83,6 +85,10 @@ const queue = new PriorityAdmissionQueue({ getMetrics().syncGlobalInflight.record(inflight); }; }, + onStartFailureRollback: () => { + inflight = Math.max(0, inflight - 1); + getMetrics().syncGlobalInflight.record(inflight); + }, onDepthChange: (depth) => getMetrics().syncBackgroundQueueDepth.record(depth), observability: { scheduler: 'sync-global', @@ -136,7 +142,6 @@ function acquire( source: SyncAdmissionSource; signal?: AbortSignal; agingThresholdMs: number; - now: () => number; }, ): PriorityAdmission { const { limit } = policy; @@ -158,7 +163,6 @@ function acquire( priorityClass: options.priorityClass, signal: options.signal, agingThresholdMs: options.agingThresholdMs, - now: options.now, queueLimit, createBusyError: () => new SyncBackpressureBusyError( `Sync backpressure rejected ${options.label} ` @@ -246,7 +250,6 @@ export function resolveSyncGlobalBackpressure( export function getSyncBackpressureSnapshot( policy?: SyncGlobalBackpressurePolicy, - now = Date.now(), ): SyncBackpressureSnapshot { const queuedByPriorityClass: Record = { elevated: 0, @@ -260,7 +263,7 @@ export function getSyncBackpressureSnapshot( limit: policy ? policy.limit ?? null : lastLimit, queueLimit: policy ? policy.queueLimit ?? null : lastQueueLimit, queuedByPriorityClass, - oldestQueuedAgeMs: queue.oldestAgeMs(now), + oldestQueuedAgeMs: queue.oldestAgeMs(), }; } @@ -282,9 +285,7 @@ export async function withGlobalSyncBackpressure( */ source?: SyncAdmissionSource; signal?: AbortSignal; - /** Deterministic scheduler injection; not operator configuration. */ agingThresholdMs?: number; - now?: () => number; logInfo?: (ctx: OperationContext, message: string) => void; }, work: () => Promise, @@ -314,7 +315,6 @@ export async function withGlobalSyncBackpressure( source: normalizeSyncAdmissionSource(options.source), signal: options.signal, agingThresholdMs: options.agingThresholdMs ?? DEFAULT_SYNC_PRIORITY_AGING_MS, - now: options.now ?? Date.now, }); } catch (error) { if (error instanceof SyncBackpressureBusyError) { diff --git a/packages/agent/src/sync/exact-assets.ts b/packages/agent/src/sync/exact-assets.ts index 61f23f3153..b0474231fc 100644 --- a/packages/agent/src/sync/exact-assets.ts +++ b/packages/agent/src/sync/exact-assets.ts @@ -1,8 +1,37 @@ -import { parseDeterministicKnowledgeAssetUal } from '@origintrail-official/dkg-core'; +import { + DKG_GOSSIP_MAX_MESSAGE_BYTES, + parseDeterministicKnowledgeAssetUal, +} from '@origintrail-official/dkg-core'; /** One VM reconciliation slice deliberately fetches at most this many KAs. */ export const MAX_EXACT_SYNC_ASSETS = 10; +/** + * A published assertion must already fit one DKG gossip application payload. + * Apply that same per-asset wire ceiling to each compatibility phase so a + * legacy responder that ignores the additive exact filter cannot turn a + * narrow repair into an unbounded full-CG accumulation. + */ +export const MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET = DKG_GOSSIP_MAX_MESSAGE_BYTES; +export const MAX_EXACT_SYNC_PHASE_QUADS_PER_ASSET = 100_000; + +export function exactSyncPhaseAccumulationLimits(assetUals: readonly string[]): { + maxBytes: number; + maxQuads: number; +} { + const assetCount = requireExactAssetUals(assetUals).length; + return { + maxBytes: assetCount * MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + // Align with the existing bounded exact-graph read contract so compact + // wire data cannot expand into an unbounded retained JS object graph. + maxQuads: assetCount * MAX_EXACT_SYNC_PHASE_QUADS_PER_ASSET, + }; +} + +function canonicalExactAssetSetOrder(assetUals: readonly string[]): string[] { + return [...new Set(assetUals)].sort(); +} + /** * Normalize the additive exact-asset sync filter. * @@ -30,7 +59,7 @@ export function normalizeExactAssetUals(value: unknown): string[] | undefined { return []; } } - return normalized; + return canonicalExactAssetSetOrder(normalized); } export function requireExactAssetUals(value: unknown): string[] { @@ -43,11 +72,13 @@ export function requireExactAssetUals(value: unknown): string[] { /** Stable identity for checkpoints, single-flight keys, and responder plans. */ export function exactAssetFilterKey(assetUals: readonly string[] | undefined): string { - return assetUals === undefined ? 'full' : `exact:${assetUals.join('\u001f')}`; + return assetUals === undefined + ? 'full' + : `exact:${canonicalExactAssetSetOrder(assetUals).join('\u001f')}`; } export function encodeExactAssetUals(assetUals: readonly string[]): string { - return encodeURIComponent(JSON.stringify(assetUals)); + return encodeURIComponent(JSON.stringify(canonicalExactAssetSetOrder(assetUals))); } export function decodeExactAssetUals(encoded: string): string[] { diff --git a/packages/agent/src/sync/priority-admission-queue.ts b/packages/agent/src/sync/priority-admission-queue.ts index 74d802f35a..d18fa3a023 100644 --- a/packages/agent/src/sync/priority-admission-queue.ts +++ b/packages/agent/src/sync/priority-admission-queue.ts @@ -1,3 +1,4 @@ +import { performance } from 'node:perf_hooks'; import { backpressureRegistry, getMetrics, @@ -20,7 +21,6 @@ export interface PriorityAdmissionEntry extends PriorityAdmissionSchedu ownerKey: string; sequence: number; enqueuedAt: number; - now: () => number; agingThresholdMs: number; } @@ -49,13 +49,19 @@ export interface PriorityAdmission { export interface PriorityAdmissionQueueHooks { canRun: (entry: PriorityAdmissionEntry) => boolean; onStart: (entry: PriorityAdmissionEntry) => PriorityAdmissionRelease; + /** Undo capacity claimed by onStart when it throws before returning its release. */ + onStartFailureRollback?: ( + entry: PriorityAdmissionEntry, + error: unknown, + ) => void; onDepthChange?: (depth: number) => void; + /** Queue-wide elapsed-time source; production defaults to a monotonic clock. */ + now?: () => number; observability?: { scheduler: string; operation: (entry: PriorityAdmissionEntry) => string; inflightLimit?: (entry: PriorityAdmissionEntry) => number | null; thresholds?: SchedulerPressureThresholds; - now?: () => number; register?: boolean; }; } @@ -68,7 +74,6 @@ export interface PriorityAdmissionAcquireOptions extends PriorityAdmiss signal?: AbortSignal; timeoutMs?: number; agingThresholdMs: number; - now?: () => number; /** Reserve one bounded queue slot if this running stage may hand off. */ reserveForHandoff?: boolean; createBusyError: (reason: 'global_queue_full' | 'owner_queue_full') => Error; @@ -104,15 +109,19 @@ export class PriorityAdmissionQueue extends ObservableScheduler { private readonly handoffReservations = new Map(); private readonly pressureTickets = new WeakMap, SchedulerPressureTicket>(); private readonly hooks: PriorityAdmissionQueueHooks; + private readonly now: () => number; private nextSequence = 0; + private agedTurnOwed = false; constructor(hooks: PriorityAdmissionQueueHooks) { + const now = hooks.now ?? (() => performance.now()); super({ scheduler: hooks.observability?.scheduler ?? 'priority-admission', thresholds: hooks.observability?.thresholds, - now: hooks.observability?.now, + now, }); this.hooks = hooks; + this.now = now; if (hooks.observability?.register) backpressureRegistry.register(this); } @@ -136,9 +145,9 @@ export class PriorityAdmissionQueue extends ObservableScheduler { return count; } - oldestAgeMs(now = Date.now()): number { + oldestAgeMs(): number { if (this.queue.length === 0) return 0; - return Math.max(0, now - Math.min(...this.queue.map((entry) => entry.enqueuedAt))); + return Math.max(0, this.now() - Math.min(...this.queue.map((entry) => entry.enqueuedAt))); } acquire(options: PriorityAdmissionAcquireOptions): PriorityAdmission { @@ -150,7 +159,7 @@ export class PriorityAdmissionQueue extends ObservableScheduler { handoffReservation?: HandoffReservation, ): PriorityAdmission { if (options.signal?.aborted) throw abortError(options.signal.reason); - const now = options.now ?? Date.now; + this.reconcileAgedTurnOwed(); const ownerKey = handoffReservation?.ownerKey ?? options.ownerKey ?? ''; const queuedBefore = this.queue.length; const sequence = handoffReservation?.sequence ?? this.nextSequence++; @@ -161,8 +170,7 @@ export class PriorityAdmissionQueue extends ObservableScheduler { priority: options.priority, priorityClass: options.priorityClass, sequence, - enqueuedAt: now(), - now, + enqueuedAt: this.now(), agingThresholdMs: options.agingThresholdMs, }; if (this.hooks.observability) { @@ -183,22 +191,32 @@ export class PriorityAdmissionQueue extends ObservableScheduler { && options.ownerQueueLimit !== undefined && this.countOwner(ownerKey) + reservedOwner >= options.ownerQueueLimit, ); + const queuedRunnable = this.queue.some((entry) => ( + !entry.settled && this.hooks.canRun(entry) + )); if ( !handoffReservation && this.hooks.canRun(base) - && queuedBefore === 0 + && !queuedRunnable && !reservationGlobalFull && !reservationOwnerFull ) { - this.recordDecision(base, 'started'); this.observePressureEnqueue(base); + let release: PriorityAdmissionRelease; + try { + release = this.start(base, options); + } catch (error) { + this.observePressureReject(base, 'start_failed'); + throw error; + } + this.recordDecision(base, 'started'); const admission: PriorityAdmission = { status: 'running', queuedBefore, sequence, entry: base, - release: Promise.resolve(this.start(base, options)), + release: Promise.resolve(release), }; if (options.reserveForHandoff) { admission.handoff = (handoffOptions) => this.handoff(base, handoffOptions); @@ -210,12 +228,7 @@ export class PriorityAdmissionQueue extends ObservableScheduler { const ownerFull = options.ownerQueueLimit !== undefined && this.countOwner(ownerKey) + reservedOwner >= options.ownerQueueLimit; if (!handoffReservation && (globalFull || ownerFull)) { - const victim = this.queue - .filter((entry) => ( - entry.priority < options.priority - && (!ownerFull || entry.ownerKey === ownerKey) - )) - .sort((a, b) => a.priority - b.priority || b.sequence - a.sequence)[0]; + const victim = this.selectDisplacementVictim(options, ownerKey, ownerFull); if (!victim) { this.recordDecision(base, 'rejected'); this.observePressureReject( @@ -252,6 +265,7 @@ export class PriorityAdmissionQueue extends ObservableScheduler { internal.timer = setTimeout(() => { if (!this.remove(internal)) return; this.observePressureReject(internal, 'queue_wait_timeout'); + this.recordDecision(internal, 'rejected'); this.rejectOnce( internal, options.createTimeoutError?.() ?? options.createBusyError('global_queue_full'), @@ -266,7 +280,7 @@ export class PriorityAdmissionQueue extends ObservableScheduler { }; this.observePressureEnqueue(internal); this.queue.push(internal); - this.depthChanged(); + this.queueChanged(); if (options.signal) { options.signal.addEventListener('abort', internal.onAbort, { once: true }); if (options.signal.aborted) internal.onAbort(); @@ -294,45 +308,126 @@ export class PriorityAdmissionQueue extends ObservableScheduler { this.cleanup(entry); this.depthChanged(); if (entry.settled) continue; + let release: PriorityAdmissionRelease; + try { + release = this.start(entry, entry); + } catch (error) { + this.observePressureReject(entry, 'start_failed'); + this.recordDecision(entry, 'rejected'); + this.rejectOnce(entry, error instanceof Error ? error : new Error(String(error))); + this.reconcileAgedTurnOwed(); + continue; + } entry.settled = true; - const waitMs = Math.max(0, entry.now() - entry.enqueuedAt); + if (selected.servesDebt) this.agedTurnOwed = false; + else if (selected.createsDebt) this.agedTurnOwed = true; + this.reconcileAgedTurnOwed(); + const waitMs = Math.max(0, this.now() - entry.enqueuedAt); getMetrics().syncSchedulerQueueWaitMs.record(waitMs, this.metricAttributes(entry)); this.recordDecision(entry, 'started'); if (selected.aged) this.recordDecision(entry, 'aged'); - entry.resolve(this.start(entry, entry)); + entry.resolve(release); } } - private selectNext(): { index: number; aged: boolean } | undefined { + private selectNext(): { + index: number; + aged: boolean; + createsDebt: boolean; + servesDebt: boolean; + } | undefined { + this.reconcileAgedTurnOwed(); + const now = this.now(); const runnable = this.queue .map((entry, index) => ({ entry, index })) .filter(({ entry }) => !entry.settled && this.hooks.canRun(entry)); if (runnable.length === 0) return undefined; const aged = runnable - .filter(({ entry }) => entry.now() - entry.enqueuedAt >= entry.agingThresholdMs) + .filter(({ entry }) => this.isAged(entry, now)) .sort((a, b) => a.entry.sequence - b.entry.sequence)[0]; - if (aged) return { index: aged.index, aged: true }; + if (this.agedTurnOwed && aged) { + return { + index: aged.index, + aged: true, + createsDebt: false, + servesDebt: true, + }; + } const highest = runnable.sort((a, b) => ( b.entry.priority - a.entry.priority || a.entry.sequence - b.entry.sequence ))[0]; - return highest ? { index: highest.index, aged: false } : undefined; + if (!highest) return undefined; + const createsDebt = !this.agedTurnOwed && this.queue.some((entry) => ( + !entry.settled + && entry !== highest.entry + && entry.priority < highest.entry.priority + && this.isAged(entry, now) + )); + return { + index: highest.index, + aged: this.isAged(highest.entry, now), + createsDebt, + servesDebt: false, + }; + } + + private isAged(entry: PriorityAdmissionEntry, now = this.now()): boolean { + return now - entry.enqueuedAt >= entry.agingThresholdMs; + } + + private reconcileAgedTurnOwed(): void { + if (!this.agedTurnOwed) return; + const now = this.now(); + if (!this.queue.some((entry) => !entry.settled && this.isAged(entry, now))) { + this.agedTurnOwed = false; + } + } + + private selectDisplacementVictim( + options: PriorityAdmissionAcquireOptions, + ownerKey: string, + ownerFull: boolean, + ): InternalEntry | undefined { + const candidates = this.queue.filter((entry) => ( + entry.priority < options.priority + && (!ownerFull || entry.ownerKey === ownerKey) + )); + const now = this.now(); + const protectedAged = candidates + .filter((entry) => this.isAged(entry, now)) + .sort((a, b) => a.sequence - b.sequence)[0]; + return candidates + .filter((entry) => entry !== protectedAged) + .sort((a, b) => a.priority - b.priority || b.sequence - a.sequence)[0]; } private start( entry: PriorityAdmissionEntry, options: Pick, 'reserveForHandoff' | 'queueLimit' | 'ownerQueueLimit'>, ): PriorityAdmissionRelease { - if (options.reserveForHandoff) { - this.handoffReservations.set(entry.sequence, { - sequence: entry.sequence, - ownerKey: entry.ownerKey, - queueLimit: options.queueLimit, - ownerQueueLimit: options.ownerQueueLimit, - }); + let release: PriorityAdmissionRelease | undefined; + try { + release = this.hooks.onStart(entry); + if (options.reserveForHandoff) { + this.handoffReservations.set(entry.sequence, { + sequence: entry.sequence, + ownerKey: entry.ownerKey, + queueLimit: options.queueLimit, + ownerQueueLimit: options.ownerQueueLimit, + }); + } + this.observePressureStart(entry); + } catch (error) { + this.handoffReservations.delete(entry.sequence); + try { + if (release) release(); + else this.hooks.onStartFailureRollback?.(entry, error); + } catch { + // Preserve the admission failure; release is best-effort rollback here. + } + throw error; } - this.observePressureStart(entry); - const release = this.hooks.onStart(entry); let released = false; return () => { if (released) return; @@ -367,7 +462,7 @@ export class PriorityAdmissionQueue extends ObservableScheduler { if (index < 0) return false; this.queue.splice(index, 1); this.cleanup(entry); - this.depthChanged(); + this.queueChanged(); return true; } @@ -388,6 +483,11 @@ export class PriorityAdmissionQueue extends ObservableScheduler { this.hooks.onDepthChange?.(this.queue.length); } + private queueChanged(): void { + this.reconcileAgedTurnOwed(); + this.depthChanged(); + } + private metricAttributes(entry: PriorityAdmissionScheduling) { return { lane: entry.lane, priority_class: entry.priorityClass }; } diff --git a/packages/agent/src/sync/requester/durable-sync.ts b/packages/agent/src/sync/requester/durable-sync.ts index 504d6cfaaf..3d62ef6d69 100644 --- a/packages/agent/src/sync/requester/durable-sync.ts +++ b/packages/agent/src/sync/requester/durable-sync.ts @@ -32,6 +32,12 @@ import { normalizeDurableSyncContext, type LegacyDurableSyncContext, } from './durable-sync-compat.js'; +import { + classifyExactDurableFetch, + filterExactAssetDurablePayload, + mergeExactDurableFetchDisposition, + type ExactDurableFetchDisposition, +} from './exact-durable-fetch.js'; export { createContextGraphSyncDeadline, @@ -47,6 +53,14 @@ export type { DurableSyncContextGraphBudgetRequest, } from './durable-sync-budget.js'; export type { LegacyDurableSyncContext } from './durable-sync-compat.js'; +export { filterExactAssetDurablePayload } from './exact-durable-fetch.js'; +export type { ExactDurableFetchDisposition } from './exact-durable-fetch.js'; + +export interface DetailedDurableSyncResult { + readonly result: InitializedDurableSyncResult; + /** Present only when this physical run used an exact-asset filter. */ + readonly exactFetchDisposition?: ExactDurableFetchDisposition; +} const DKG_NS = 'http://dkg.io/ontology/'; const CONTENT_SCOPE_VERSION = `${DKG_NS}contentScopeVersion`; @@ -187,39 +201,6 @@ export interface DurableSyncContext { logDebug: (ctx: OperationContext, message: string) => void; } -/** - * Rolling-upgrade guard: an old responder may ignore the additive exact-asset - * filter and return the whole CG. Keep only requested descriptor subjects and - * their declared assertion graphs before any verification or store write. - */ -export function filterExactAssetDurablePayload( - dataQuads: readonly Quad[], - metaQuads: readonly Quad[], - assetUals: readonly string[], -): { dataQuads: Quad[]; metaQuads: Quad[]; descriptorCoverageComplete: boolean } { - const exactUals = new Set(assetUals); - const exactMeta = metaQuads.filter((quad) => exactUals.has(quad.subject)); - const returnedDescriptors = new Set( - exactMeta - .filter((quad) => ( - quad.predicate === KA_UAL - && quad.subject === stripLiteral(quad.object) - )) - .map((quad) => quad.subject), - ); - const exactGraphs = new Set( - exactMeta - .filter((quad) => quad.predicate === ASSERTION_GRAPH) - .map((quad) => quad.object), - ); - return { - metaQuads: exactMeta, - dataQuads: dataQuads.filter((quad) => exactGraphs.has(quad.graph)), - descriptorCoverageComplete: returnedDescriptors.size === exactUals.size - && [...exactUals].every((ual) => returnedDescriptors.has(ual)), - }; -} - export function runDurableSync( context: DurableSyncContext, ): Promise; @@ -229,12 +210,24 @@ export function runDurableSync( export async function runDurableSync( context: DurableSyncContext | LegacyDurableSyncContext, ): Promise { + return (await runDurableSyncWithBudget(normalizeDurableSyncContext(context))).result; +} + +export function runDurableSyncDetailed( + context: DurableSyncContext, +): Promise; +export function runDurableSyncDetailed( + context: LegacyDurableSyncContext, +): Promise; +export async function runDurableSyncDetailed( + context: DurableSyncContext | LegacyDurableSyncContext, +): Promise { return runDurableSyncWithBudget(normalizeDurableSyncContext(context)); } async function runDurableSyncWithBudget( context: DurableSyncContext, -): Promise { +): Promise { const { ctx, remotePeerId, @@ -273,6 +266,7 @@ async function runDurableSyncWithBudget( }); const accumulator = createDurableSyncAccumulator(); + const exactFetchDispositions: ExactDurableFetchDisposition[] = []; const recordPhaseOutcome = ( result: SyncPageResult, @@ -323,6 +317,7 @@ async function runDurableSyncWithBudget( for (const [contextGraphIndex, pid] of contextGraphIds.entries()) { let activePhase: 'fetch' | 'verify' | 'store' | undefined; let peerRespondedForContextGraph = false; + let exactFetchDispositionIndex: number | undefined; const startPhase = (phase: 'fetch' | 'verify' | 'store') => { activePhase = phase; onPhase?.(phase, 'start'); @@ -344,6 +339,9 @@ async function runDurableSyncWithBudget( const deadline = contextGraphBudget.fetchDeadline; const activeFetchContext = fetchContext(deadline); const exactAssetUals = exactAssetUalsFor?.(pid); + if (exactAssetUals !== undefined) { + exactFetchDispositionIndex = exactFetchDispositions.push('incomplete') - 1; + } logInfo(ctx, `Syncing context graph "${pid}" from ${remotePeerId}`); @@ -564,6 +562,17 @@ async function runDurableSyncWithBudget( && !metaResult.timedOut && effectiveDataResult.completed && !effectiveDataResult.timedOut; + const settledExactDisposition = (): ExactDurableFetchDisposition => ( + classifyExactDurableFetch({ + requestedAssetCount: exactAssetUals?.length ?? 0, + metaResult, + dataResult: rawDataResult, + metaFetched: !skipAgentsMeta, + descriptorCoverageComplete: exactAssetDescriptorCoverageComplete, + rejectedKcs: processed.rejectedKcs, + dataRejectedMissingMeta: processed.dataRejectedMissingMeta, + }) + ); // Metadata-only pages may move the meta cursor after storage, but they // still are not usable data progress for freshness/backoff accounting. if ( @@ -588,6 +597,9 @@ async function runDurableSyncWithBudget( emptyPhase, }); markDurableTerminalBoundary(accumulator, reachedContextGraphTerminalBoundary); + if (exactFetchDispositionIndex !== undefined) { + exactFetchDispositions[exactFetchDispositionIndex] = settledExactDisposition(); + } if ((metaResult.timedOut || effectiveDataResult.timedOut) && shouldStopAfterBackoffWorthyFailure(pid, 'phase timeout')) { break; } @@ -666,6 +678,9 @@ async function runDurableSyncWithBudget( recordPhaseOutcome(metaResult, { updateCheckpoint: updateMetaCheckpoint, countProgress: !metadataOnlyResponse }); recordPhaseOutcome(effectiveDataResult, { updateCheckpoint: updateDataCheckpoint }); markDurableTerminalBoundary(accumulator, reachedContextGraphTerminalBoundary); + if (exactFetchDispositionIndex !== undefined) { + exactFetchDispositions[exactFetchDispositionIndex] = settledExactDisposition(); + } endPhase(); if ((metaResult.timedOut || effectiveDataResult.timedOut) && shouldStopAfterBackoffWorthyFailure(pid, 'phase timeout')) { break; @@ -719,8 +734,15 @@ async function runDurableSyncWithBudget( if (result.insertedTriples > 0) { logInfo(ctx, `Sync complete: ${result.insertedTriples} verified triples from ${remotePeerId}`); } + const exactFetchDisposition = exactFetchDispositions.reduce( + mergeExactDurableFetchDisposition, + undefined, + ); - return result; + return { + result, + ...(exactFetchDisposition ? { exactFetchDisposition } : {}), + }; } function partitionVerifiedGraphScopedAssets( diff --git a/packages/agent/src/sync/requester/exact-durable-fetch.ts b/packages/agent/src/sync/requester/exact-durable-fetch.ts new file mode 100644 index 0000000000..894dd20995 --- /dev/null +++ b/packages/agent/src/sync/requester/exact-durable-fetch.ts @@ -0,0 +1,88 @@ +import type { Quad } from '@origintrail-official/dkg-storage'; +import type { SyncPageResult } from './page-fetch.js'; +import { stripLiteral } from '../../dkg-agent-utils.js'; + +const DKG_NS = 'http://dkg.io/ontology/'; +const KA_UAL = `${DKG_NS}kaUal`; +const ASSERTION_GRAPH = `${DKG_NS}assertionGraph`; + +export type ExactDurableFetchDisposition = 'found' | 'clean-absent' | 'incomplete'; + +/** + * Rolling-upgrade guard: an old responder may ignore the additive exact-asset + * filter and return the whole CG. Keep only requested descriptor subjects and + * their declared assertion graphs before any verification or store write. + */ +export function filterExactAssetDurablePayload( + dataQuads: readonly Quad[], + metaQuads: readonly Quad[], + assetUals: readonly string[], +): { dataQuads: Quad[]; metaQuads: Quad[]; descriptorCoverageComplete: boolean } { + const exactUals = new Set(assetUals); + const exactMeta = metaQuads.filter((quad) => exactUals.has(quad.subject)); + const returnedDescriptors = new Set( + exactMeta + .filter((quad) => ( + quad.predicate === KA_UAL + && quad.subject === stripLiteral(quad.object) + )) + .map((quad) => quad.subject), + ); + const exactGraphs = new Set( + exactMeta + .filter((quad) => quad.predicate === ASSERTION_GRAPH) + .map((quad) => quad.object), + ); + return { + metaQuads: exactMeta, + dataQuads: dataQuads.filter((quad) => exactGraphs.has(quad.graph)), + descriptorCoverageComplete: returnedDescriptors.size === exactUals.size + && [...exactUals].every((ual) => returnedDescriptors.has(ual)), + }; +} + +export function classifyExactDurableFetch(params: { + requestedAssetCount: number; + metaResult: SyncPageResult; + dataResult: SyncPageResult; + metaFetched: boolean; + descriptorCoverageComplete: boolean; + rejectedKcs: number; + dataRejectedMissingMeta: number; +}): ExactDurableFetchDisposition { + const cleanPhase = (phase: SyncPageResult) => ( + phase.completed + && !phase.timedOut + && phase.nextOffset >= phase.resumedFromOffset + ); + if ( + params.requestedAssetCount === 0 + || !params.metaFetched + || !cleanPhase(params.metaResult) + || !cleanPhase(params.dataResult) + || params.rejectedKcs !== 0 + || params.dataRejectedMissingMeta !== 0 + ) return 'incomplete'; + + const freshEmptyPhase = (phase: SyncPageResult) => ( + phase.responderSessionStartedFresh === true + && phase.resumedFromOffset === 0 + && phase.nextOffset === 0 + && phase.quads.length === 0 + ); + if (freshEmptyPhase(params.metaResult) && freshEmptyPhase(params.dataResult)) { + return 'clean-absent'; + } + + return params.descriptorCoverageComplete ? 'found' : 'incomplete'; +} + +export function mergeExactDurableFetchDisposition( + current: ExactDurableFetchDisposition | undefined, + next: ExactDurableFetchDisposition, +): ExactDurableFetchDisposition { + if (current === undefined) return next; + if (current === 'incomplete' || next === 'incomplete') return 'incomplete'; + if (current === 'found' || next === 'found') return 'found'; + return 'clean-absent'; +} diff --git a/packages/agent/src/sync/requester/graph-scoped-materialization.ts b/packages/agent/src/sync/requester/graph-scoped-materialization.ts index fb0d92efdd..3adb8d1689 100644 --- a/packages/agent/src/sync/requester/graph-scoped-materialization.ts +++ b/packages/agent/src/sync/requester/graph-scoped-materialization.ts @@ -310,10 +310,27 @@ export async function authenticateVerifiedGraphScopedAsset( export async function materializeVerifiedGraphScopedAsset(params: { store: TripleStore; asset: VerifiedGraphScopedAsset; + isCurrent?: () => boolean; + shouldQuarantineCommitted?: () => boolean; options?: QueryOptions; oversizeHooks?: OversizeGuardHooks; }): Promise { - const { store, asset, options = {}, oversizeHooks } = params; + const { + store, + asset, + isCurrent, + shouldQuarantineCommitted, + options = {}, + oversizeHooks, + } = params; + const assertCurrent = () => { + if (isCurrent?.() === false) { + const error = new Error(`Graph-scoped materialization lifecycle for ${asset.ual} is no longer current`); + error.name = 'AbortError'; + throw error; + } + }; + assertCurrent(); const filtered = filterOversizedSyncQuads([ ...asset.dataQuads, ...asset.metadataQuads, @@ -324,12 +341,14 @@ export async function materializeVerifiedGraphScopedAsset(params: { } return withMaterializationLock(asset.metaGraph, asset.ual, async () => { + assertCurrent(); const currentVersion = await readCurrentAssertionVersion( store, asset.metaGraph, asset.ual, options, ); + assertCurrent(); if (currentVersion !== undefined && currentVersion > asset.assertionVersion) { return 'stale'; } @@ -347,6 +366,7 @@ export async function materializeVerifiedGraphScopedAsset(params: { currentMetadata, ); } + assertCurrent(); } const locallyTrustedMetadata = await readLocallyTrustedKnowledgeAssetControls( store, @@ -355,6 +375,11 @@ export async function materializeVerifiedGraphScopedAsset(params: { replacementMetadata, options, ); + // This is the last interruptible boundary. Once the atomic replacement is + // dispatched, its real completion owns the materialization lock and stop() + // must drain it rather than detaching the writer. + assertCurrent(); + const { signal: _lifecycleSignal, ...commitOptions } = options; const replaced = await tryReplaceGraphAndSubjectAtomically( store, @@ -363,7 +388,7 @@ export async function materializeVerifiedGraphScopedAsset(params: { asset.metaGraph, asset.ual, [...replacementMetadata, ...locallyTrustedMetadata], - options, + commitOptions, ); if (!replaced) { throw Object.assign( @@ -371,8 +396,27 @@ export async function materializeVerifiedGraphScopedAsset(params: { { code: 'VM_ATOMIC_REPLACE_UNSUPPORTED' }, ); } + if (shouldQuarantineCommitted?.() === true) { + const quarantined = await tryReplaceGraphAndSubjectAtomically( + store, + asset.assertionGraph, + [], + asset.metaGraph, + asset.ual, + [], + commitOptions, + ); + if (!quarantined) { + throw Object.assign( + new Error('Graph-scoped durable sync requires atomic stale-binding quarantine support'), + { code: 'VM_ATOMIC_REPLACE_UNSUPPORTED' }, + ); + } + return 'quarantined'; + } + if (isCurrent?.() === false) return 'quarantined'; return 'applied'; - }); + }, { signal: options.signal }); } /** Read the current subject once; publisher metadata helpers own typed merging. */ diff --git a/packages/agent/src/sync/requester/page-fetch.ts b/packages/agent/src/sync/requester/page-fetch.ts index 0004b4a6c9..2682818972 100644 --- a/packages/agent/src/sync/requester/page-fetch.ts +++ b/packages/agent/src/sync/requester/page-fetch.ts @@ -112,12 +112,31 @@ export interface SyncPageResult { quads: Quad[]; bytesReceived: number; resumedFromOffset: number; + /** + * True only when this phase started without reusing a requester-side + * responder snapshot token. Optional for rolling deep-import compatibility; + * proof-sensitive callers must treat an omitted value as unknown. + */ + responderSessionStartedFresh?: boolean; nextOffset: number; checkpointKey: string; completed: boolean; timedOut: boolean; } +export class SyncPageAccumulationLimitError extends Error { + readonly code = 'SYNC_PAGE_ACCUMULATION_LIMIT' as const; + + constructor( + readonly dimension: 'bytes' | 'quads', + readonly actual: number, + readonly limit: number, + ) { + super(`Sync phase ${dimension} accumulation ${actual} exceeds limit ${limit}`); + this.name = 'SyncPageAccumulationLimitError'; + } +} + interface FetchSyncPagesParams { ctx: OperationContext; remotePeerId: string; @@ -173,6 +192,9 @@ interface FetchSyncPagesParams { sinceBatchId?: string; /** Exact KAs requested by VM recovery. Undefined retains ordinary full sync. */ assetUals?: string[]; + /** Optional cumulative ceilings for proof-sensitive narrow fetches. */ + maxAcceptedBytes?: number; + maxAcceptedQuads?: number; parseAndFilter: (nquadsText: string, graphUri: string, contextGraphId: string) => Promise<{ quads: Quad[]; totalQuads: number }>; /** * Per-attempt send hook. `DKGAgent`'s production adapter sends raw @@ -251,6 +273,8 @@ export async function fetchSyncPages(params: FetchSyncPagesParams): Promise 0 && !savedResponderSession) { checkpointStore.delete(checkpointKey); offset = 0; @@ -397,6 +422,17 @@ export async function fetchSyncPages(params: FetchSyncPagesParams): Promise maxAcceptedBytes) { + const error = new SyncPageAccumulationLimitError( + 'bytes', + nextBytesReceived, + maxAcceptedBytes, + ); + markSyncPeerResponded(error); + throw error; + } + let parsed: { quads: Quad[]; totalQuads: number }; let decodeDurationMs = 0; let parseDurationMs = 0; @@ -404,7 +440,7 @@ export async function fetchSyncPages(params: FetchSyncPagesParams): Promise maxAcceptedQuads) { + throw new SyncPageAccumulationLimitError( + 'quads', + nextAcceptedQuads, + maxAcceptedQuads, + ); + } } catch (error) { markSyncPeerResponded(error); throw error; @@ -468,7 +512,13 @@ export async function fetchSyncPages(params: FetchSyncPagesParams): Promise { + it('keeps one active and one latest background mutation per busy key', async () => { + const scheduler = new ContextGraphMembershipPersistScheduler(4, 4); + let releaseActive!: () => void; + let markActive!: () => void; + const activeEntered = new Promise((resolve) => { markActive = resolve; }); + const activeGate = new Promise((resolve) => { releaseActive = resolve; }); + const executed: number[] = []; + const active = scheduler.enqueue('cg\0node\0peer', async () => { + executed.push(0); + markActive(); + await activeGate; + }); + await activeEntered; + + const pending = Array.from({ length: 1_000 }, (_, index) => + scheduler.enqueue('cg\0node\0peer', async () => { executed.push(index + 1); })); + expect(scheduler.status()).toMatchObject({ lanes: 1, active: 1, pending: 1 }); + + releaseActive(); + await Promise.all([active, ...pending]); + expect(executed).toEqual([0, 1_000]); + expect(scheduler.status()).toEqual({ closed: false, lanes: 0, active: 0, pending: 0 }); + }); + + it('preserves strict FIFO writes and rejects work beyond bounded capacity', async () => { + const scheduler = new ContextGraphMembershipPersistScheduler(2, 2); + let releaseA!: () => void; + let releaseB!: () => void; + const gateA = new Promise((resolve) => { releaseA = resolve; }); + const gateB = new Promise((resolve) => { releaseB = resolve; }); + const executed: string[] = []; + const activeA = scheduler.enqueue('a', async () => { executed.push('a0'); await gateA; }); + const activeB = scheduler.enqueue('b', async () => { executed.push('b0'); await gateB; }); + await Promise.resolve(); + + const strictA1 = scheduler.enqueue('a', async () => { executed.push('a1'); }, { strict: true }); + const strictA2 = scheduler.enqueue('a', async () => { executed.push('a2'); }, { strict: true }); + await expect(scheduler.enqueue('a', async () => undefined, { strict: true })) + .rejects.toBeInstanceOf(ContextGraphMembershipPersistQueueFullError); + await expect(scheduler.enqueue('c', async () => undefined, { strict: true })) + .rejects.toBeInstanceOf(ContextGraphMembershipPersistQueueFullError); + + releaseA(); + releaseB(); + await Promise.all([activeA, activeB, strictA1, strictA2]); + expect(executed.filter((item) => item.startsWith('a'))).toEqual(['a0', 'a1', 'a2']); + }); + + it('closes admission and drains physical writes before resolving', async () => { + const scheduler = new ContextGraphMembershipPersistScheduler(); + let release!: () => void; + let markEntered!: () => void; + const entered = new Promise((resolve) => { markEntered = resolve; }); + const gate = new Promise((resolve) => { release = resolve; }); + const write = scheduler.enqueue('key', async () => { markEntered(); await gate; }); + await entered; + const drained = scheduler.closeAndDrain(); + const drainSettled = vi.fn(); + void drained.then(drainSettled); + + await expect(scheduler.enqueue('late', async () => undefined)) + .rejects.toBeInstanceOf(ContextGraphMembershipPersistQueueClosedError); + await Promise.resolve(); + expect(drainSettled).not.toHaveBeenCalled(); + + release(); + await Promise.all([write, drained]); + expect(drainSettled).toHaveBeenCalledOnce(); + expect(scheduler.status()).toEqual({ closed: true, lanes: 0, active: 0, pending: 0 }); + }); +}); diff --git a/packages/agent/test/core-fills-gap.test.ts b/packages/agent/test/core-fills-gap.test.ts index ba2e658d74..56133766fe 100644 --- a/packages/agent/test/core-fills-gap.test.ts +++ b/packages/agent/test/core-fills-gap.test.ts @@ -50,6 +50,7 @@ import type { VmReconcileNegativeRecord, } from '../src/dkg-agent-types.js'; import { DKGAgent } from '../src/index.js'; +import { DKGAgentBase } from '../src/dkg-agent-base.js'; import { VmReconcileDispatcher, type PendingOrdinalRecoveryResult, @@ -76,13 +77,17 @@ interface AgentInternals { localCgId: string, onChainCgId: bigint, targets: readonly Array<{ + localCgId: string; + onChainCgId: string; ordinal: number; ual: string; + merkleRoot: string; kaId: string; reason: 'no-swm' | 'verified-vm-metadata-pending'; }>, headBlock: number | undefined, isTargetCurrent: () => boolean, + signal?: AbortSignal, ): Promise; syncContextGraphFromConnectedPeers(contextGraphId: string, options?: { includeSharedMemory?: boolean; maxPeers?: number; peerRotationKey?: string }): Promise; runVmReconcileForCg(localCgId: string, source?: 'live' | 'periodic' | 'manual'): Promise<{ @@ -160,6 +165,22 @@ function emptyCatchupStats() { }; } +function vmRecoveryTarget( + localCgId: string, + ordinal: number, + kaId = String(ordinal), +) { + return { + localCgId, + onChainCgId: '1', + ordinal, + ual: `did:dkg:base:84532/0x0000000000000000000000000000000000000001/${kaId}`, + merkleRoot: `root-${kaId}`, + kaId, + reason: 'no-swm' as const, + }; +} + function noProtocolCatchupStats() { return { ...emptyCatchupStats(), @@ -443,13 +464,22 @@ describe('Phase D — recordCoreHostedPublicCg', () => { const internals = await boot(); internals.chain.getContextGraphAccessPolicy = async () => new Promise(() => undefined); (internals.chain as { isContextGraphActiveOnChain?: unknown }).isContextGraphActiveOnChain = undefined; - - const startedAt = Date.now(); - await internals.recordCoreHostedPublicCg('44'); - - expect(Date.now() - startedAt).toBeLessThan(5_000); - expect(internals.subscribedContextGraphs.get('44')).toBeUndefined(); - expect(saved.find((r) => r.id === '44')).toBeUndefined(); + vi.useFakeTimers(); + try { + let settled = false; + const recording = internals.recordCoreHostedPublicCg('44') + .finally(() => { settled = true; }); + await vi.advanceTimersByTimeAsync(2_499); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await recording; + + expect(settled).toBe(true); + expect(internals.subscribedContextGraphs.get('44')).toBeUndefined(); + expect(saved.find((r) => r.id === '44')).toBeUndefined(); + } finally { + vi.useRealTimers(); + } }); it('stops accepting and drains core-host recordings deterministically', async () => { @@ -631,6 +661,39 @@ describe('Phase D — recordCoreHostedPublicCg', () => { expect(((internals as any).recentReconciledUals as { has(key: string): boolean }).has(recentKey)).toBe(false); }); + it('reclaims binding generations when subscription records are deleted', async () => { + const internals = await boot(); + const generations = (internals as any).contextGraphBindingGenerations as Map; + + for (let index = 0; index < 32; index += 1) { + const localCgId = `deleted-binding-${index}`; + const subscription = { subscribed: true }; + internals.subscribedContextGraphs.set(localCgId, subscription); + (internals as any).bindSubscriptionOnChainId(localCgId, subscription, String(index + 1)); + expect((internals as any).deleteContextGraphSubscription(localCgId)).toBe(true); + } + + expect(generations.size).toBe(0); + + const reusedCgId = 'deleted-binding-reused'; + const oldSubscription = { subscribed: true }; + internals.subscribedContextGraphs.set(reusedCgId, oldSubscription); + (internals as any).bindSubscriptionOnChainId(reusedCgId, oldSubscription, '100'); + const oldGeneration = (internals as any).captureContextGraphBindingGeneration(reusedCgId); + (internals as any).deleteContextGraphSubscription(reusedCgId); + const replacement = { subscribed: true }; + internals.subscribedContextGraphs.set(reusedCgId, replacement); + (internals as any).bindSubscriptionOnChainId(reusedCgId, replacement, '101'); + + expect( + internals.subscribedContextGraphs.get(reusedCgId) === oldSubscription + && (internals as any).isContextGraphBindingGenerationCurrent( + reusedCgId, + oldGeneration, + ), + ).toBe(false); + }); + it('clears VM reconcile state when stale inactive on-chain ids are re-registered', async () => { const internals = await boot(); const localCgId = 'stale-register'; @@ -814,6 +877,77 @@ describe('Phase D - VM reconcile damping', () => { }); }); + it('clears exact-recovery rotation state on the direct recent-cache terminal path', async () => { + const internals = await boot(); + const localCgId = '68'; + const onChainCgId = 68n; + const kaId = 9068n; + registerUnmatchedKC(internals.chain, kaId, onChainCgId); + const storageAddress = await internals.chain.getDKGKnowledgeAssetsAddress(); + const ual = buildKnowledgeAssetUal(internals.chain.chainId, storageAddress, kaId); + const merkleRoot = await internals.chain.getLatestMerkleRoot(kaId); + const target = { + localCgId, + onChainCgId: onChainCgId.toString(), + ordinal: 0, + ual, + merkleRoot: bytesToHex(merkleRoot), + kaId: kaId.toString(), + reason: 'no-swm' as const, + }; + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + (internals as any).prepareVmReconcileRotationTarget( + target, ['12D3KooWDirectTerminalRecent'], 100, + ); + ((internals as any).recentReconciledUals as { add(key: string): void }).add( + (internals as any).vmReconcileCacheKey(localCgId, ual, merkleRoot), + ); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(true); + + await expect(internals.reconcileChainOrdinal(localCgId, onChainCgId, 0, undefined)) + .resolves.toEqual({ status: 'already', blockNumber: 0 }); + + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + }); + + it.each([ + ['promoted', 'reconciled'], + ['already-confirmed', 'already'], + ['stale-target', 'already'], + ] as const)( + 'clears exact-recovery rotation state on direct %s finalization', + async (finalizationOutcome, expectedStatus) => { + const internals = await boot(); + const ordinalByOutcome = { + promoted: 69, + 'already-confirmed': 70, + 'stale-target': 71, + } as const; + const graphOrdinal = ordinalByOutcome[finalizationOutcome]; + const localCgId = String(graphOrdinal); + const onChainCgId = BigInt(graphOrdinal); + const kaId = BigInt(9_000 + graphOrdinal); + registerUnmatchedKC(internals.chain, kaId, onChainCgId); + const target = { + ...vmRecoveryTarget(localCgId, 0, kaId.toString()), + onChainCgId: onChainCgId.toString(), + }; + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + (internals as any).prepareVmReconcileRotationTarget( + target, [`12D3KooWDirectTerminal${graphOrdinal}`], 100, + ); + (internals as any).getOrCreateFinalizationHandler = () => ({ + handleChainReconciledKC: async () => finalizationOutcome, + }); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(true); + + await expect(internals.reconcileChainOrdinal(localCgId, onChainCgId, 0, undefined)) + .resolves.toMatchObject({ status: expectedStatus }); + + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + }, + ); + it('negative-caches a missing SWM snapshot and skips the expensive scan plus active fetch during backoff', async () => { const internals = await boot(); const onChainCgId = 42n; @@ -1770,6 +1904,37 @@ describe('Phase D - VM reconcile damping', () => { expect(negativeCache.get(cacheKey)?.nextRetryAt).toBeGreaterThan(now); }); + it('bounds durable negative-cache hydration guards and safely reloads evicted keys', async () => { + const loads: string[] = []; + const internals = await boot({ + loadAll: async () => [], + save: async () => undefined, + delete: async () => undefined, + loadVmReconcileNegative: async (cacheKey) => { + loads.push(cacheKey); + return undefined; + }, + }); + const hydrated = (internals as any).vmReconcileNegativeCacheHydrated as Map; + const cap = DKGAgent.VM_RECONCILE_CACHE_MAX_ENTRIES; + + for (let index = 0; index < cap + 2; index += 1) { + (internals as any).markVmReconcileNegativeCacheHydrated( + `hydrated-cg\0ual-${index}#root`, + 'hydrated-cg', + ); + } + + expect(hydrated.size).toBe(cap); + expect(hydrated.has('hydrated-cg\0ual-0#root')).toBe(false); + await expect((internals as any).shouldDeferVmReconcileByNegativeCache( + 'hydrated-cg\0ual-0#root', + 'hydrated-cg', + )).resolves.toBe(false); + expect(loads).toEqual(['hydrated-cg\0ual-0#root']); + expect(hydrated.size).toBe(cap); + }); + it('prunes oversized VM reconcile state and clears non-hosted CG state on unsubscribe', async () => { const internals = await boot(); const negativeCache = (internals as any).vmReconcileNegativeCache as Map { const fetchCooldown = (internals as any).vmReconcileFetchCooldownAt as Map; const peerCursor = (internals as any).vmReconcileCatchupPeerCursor as Map; const peerOrder = (internals as any).vmReconcileCatchupPeerOrder as Map; + const rotationState = (internals as any).vmReconcileRotationState as Map; + const hydrated = (internals as any).vmReconcileNegativeCacheHydrated as Map; const recent = (internals as any).recentReconciledUals as { add(key: string): void; has(key: string): boolean }; const now = Date.now(); @@ -1820,15 +1987,22 @@ describe('Phase D - VM reconcile damping', () => { peerTopologyKey: '', }); (internals as any).indexVmReconcileNegativeCacheEntry('cleanup-cg', 'cleanup-cache'); + (internals as any).markVmReconcileNegativeCacheHydrated('cleanup-hydrated', 'cleanup-cg'); fetchCooldown.set('cleanup-cg', now); peerCursor.set('cleanup-cg', 7); peerOrder.set('cleanup-cg', { orderedPeers: ['peer-a'], nextPeerId: 'peer-a' }); + const cleanupRotationKey = (internals as any).vmReconcileRotationSlotKey( + vmRecoveryTarget('cleanup-cg', 0), + ); + rotationState.set(cleanupRotationKey, {}); recent.add('cleanup-cg\0did:dkg:mock:31337/0x000000000000000000000000000000000000c10a/1#01'); (agent as any).unsubscribeFromContextGraph('cleanup-cg'); expect(negativeCache.has('cleanup-cache')).toBe(false); + expect(hydrated.has('cleanup-hydrated')).toBe(false); expect(fetchCooldown.has('cleanup-cg')).toBe(false); expect(peerCursor.has('cleanup-cg')).toBe(false); expect(peerOrder.has('cleanup-cg')).toBe(false); + expect(rotationState.has(cleanupRotationKey)).toBe(false); expect(recent.has('cleanup-cg\0did:dkg:mock:31337/0x000000000000000000000000000000000000c10a/1#01')).toBe(false); internals.subscribedContextGraphs.set('hosted-cg', { subscribed: true, coreHosted: true }); @@ -1841,15 +2015,22 @@ describe('Phase D - VM reconcile damping', () => { peerTopologyKey: '', }); (internals as any).indexVmReconcileNegativeCacheEntry('hosted-cg', 'hosted-cache'); + (internals as any).markVmReconcileNegativeCacheHydrated('hosted-hydrated', 'hosted-cg'); fetchCooldown.set('hosted-cg', now); peerCursor.set('hosted-cg', 3); peerOrder.set('hosted-cg', { orderedPeers: ['peer-b'], nextPeerId: 'peer-b' }); + const hostedRotationKey = (internals as any).vmReconcileRotationSlotKey( + vmRecoveryTarget('hosted-cg', 0), + ); + rotationState.set(hostedRotationKey, {}); recent.add('hosted-cg\0did:dkg:mock:31337/0x000000000000000000000000000000000000c10a/2#02'); (agent as any).unsubscribeFromContextGraph('hosted-cg'); expect(negativeCache.has('hosted-cache')).toBe(true); + expect(hydrated.has('hosted-hydrated')).toBe(true); expect(fetchCooldown.has('hosted-cg')).toBe(true); expect(peerCursor.has('hosted-cg')).toBe(true); expect(peerOrder.has('hosted-cg')).toBe(true); + expect(rotationState.has(hostedRotationKey)).toBe(false); expect(recent.has('hosted-cg\0did:dkg:mock:31337/0x000000000000000000000000000000000000c10a/2#02')).toBe(true); }); }); @@ -2060,19 +2241,22 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { return true; }; const fetches: Array<{ peerId: string; uals: string[] }> = []; - (internals as any).syncExactKnowledgeAssetsFromPeer = async ( + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( peerId: string, _cg: string, uals: string[], ) => { fetches.push({ peerId, uals }); return { - fetchedDataTriples: 1, - fetchedMetaTriples: 8, - insertedTriples: 9, - failedPeers: 0, - failedPhases: 0, - deferredBackpressure: 0, + result: { + fetchedDataTriples: 1, + fetchedMetaTriples: 8, + insertedTriples: 9, + failedPeers: 0, + failedPhases: 0, + deferredBackpressure: 0, + }, + disposition: 'found', }; }; (internals as any).reconcileChainOrdinal = async () => ({ @@ -2084,18 +2268,371 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { const result = await internals.recoverVmReconcileBatch( localCgId, 1n, - [{ ordinal: 0, ual, kaId: '7', reason: 'no-swm' }], + [{ ...vmRecoveryTarget(localCgId, 0, '7'), ual }], 100, () => true, ); - expect(connectionAttempts).toEqual([approvedPeer, registryPeer]); - expect(protocolPeers[0]).toBe(connected[0]); - expect(fetches).toEqual([{ peerId: approvedPeer, uals: [ual] }]); + expect(connectionAttempts).toEqual([]); + expect(protocolPeers[0]?.toString()).toBe(registryPeer); + expect(fetches).toEqual([{ peerId: registryPeer, uals: [ual] }]); expect(result.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); expect(result.attemptedOrdinals).toEqual([0]); }); + it('canonicalizes authoritative curators before the cap and ahead of a stale hint', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmCanonicalCurators', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/canonical-curators'; + const staleHint = '12D3KooWCanonicalCuratorHint'; + const curators = [1, 2, 3, 4].map((n) => `12D3KooWCanonicalCurator${n}`); + const connectedById = new Map( + [staleHint, ...curators].map((peerId) => [peerId, { toString: () => peerId }]), + ); + (internals as any).node = { + peerId: '12D3KooWCanonicalCuratorLocal', + libp2p: { + getConnections: () => [...connectedById.values()].map((remotePeer) => ({ remotePeer })), + }, + }; + (internals as any).preferredSyncPeers.set(localCgId, staleHint); + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [curators[3], curators[1], curators[2], curators[0]], + curatorIsLocal: false, + legacyTripleResolved: false, + }); + const connectionAttempts: string[] = []; + (internals as any).ensurePeerConnected = async (peerId: string) => { + connectionAttempts.push(peerId); + }; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const fetches: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + fetches.push(peerId); + return { + result: { + fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'found', + }; + }; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + + await internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, '85')], 100, () => true, + ); + + expect(connectionAttempts).toEqual([]); + expect(fetches).toEqual([curators[0]]); + expect((internals as any).vmReconcileCuratorPeersByCg.get(localCgId)) + .toEqual([...curators, staleHint]); + }); + + it('retains four curators and eventually recovers an asset held only by curator four', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmFourthCurator', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/fourth-curator'; + const curators = [1, 2, 3, 4].map((n) => `12D3KooWFourthCurator${n}`); + const connected = curators.map((peerId) => ({ toString: () => peerId })); + (internals as any).node = { + peerId: '12D3KooWFourthCuratorLocal', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [...curators].reverse(), curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const attempts: string[] = []; + let lastPeerId: string | undefined; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + attempts.push(peerId); + lastPeerId = peerId; + const found = peerId === curators[3]; + return { + result: { + fetchedDataTriples: found ? 1 : 0, + fetchedMetaTriples: found ? 8 : 0, + insertedTriples: found ? 9 : 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: found ? 'found' : 'clean-absent', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, 'fourth-holder'); + (internals as any).reconcileChainOrdinal = async () => ( + lastPeerId === curators[3] + ? { status: 'reconciled', blockNumber: 100 } + : { status: 'pending', recovery: target } + ); + + for (let pass = 0; pass < 3; pass += 1) { + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(result.outcomes.get(0)).toMatchObject({ status: 'pending' }); + expect((internals as any).vmReconcileRotationState.get( + (internals as any).vmReconcileRotationSlotKey(target), + )).toMatchObject({ phase: 'collecting' }); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + } + const recovered = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + + expect(attempts).toEqual(curators); + expect(recovered.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + expect((internals as any).vmReconcileCuratorPeersByCg.get(localCgId)).toEqual(curators); + }); + + it('rotates a bounded oversized-roster transport window without treating it as absence proof', async () => { + const rosterDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_EXACT_ROSTER_MAX', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_EXACT_ROSTER_MAX', { + ...rosterDescriptor, + value: 4, + }); + try { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRosterOverflow', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/roster-overflow'; + const overflowPeers = Array.from({ length: 5 }, (_, i) => `12D3KooWRosterOverflow${i}`); + const connectedById = new Map(); + (internals as any).node = { + peerId: '12D3KooWRosterOverflowLocal', + libp2p: { + getConnections: () => [...connectedById.values()].map((remotePeer) => ({ remotePeer })), + }, + }; + let resolutions = 0; + (internals as any).resolveCuratorPeerIdsForCg = async ( + _cgId: string, + options: { afterPeerId?: string }, + ) => { + resolutions += 1; + const previousIndex = options.afterPeerId + ? overflowPeers.indexOf(options.afterPeerId) + : -1; + const peerId = overflowPeers[(previousIndex + 1) % overflowPeers.length]!; + return { + peerIds: [peerId], + curatorIsLocal: false, + legacyTripleResolved: false, + overflowed: true, + nextPageAfterPeerId: peerId, + }; + }; + (internals as any).ensurePeerConnected = async (peerId: string) => { + connectedById.set(peerId, { toString: () => peerId }); + }; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const fetches: string[] = []; + const target = vmRecoveryTarget(localCgId, 0, 'roster-overflow'); + const holderPeerId = overflowPeers[4]!; + let lastPeerId: string | undefined; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + fetches.push(peerId); + lastPeerId = peerId; + const found = peerId === holderPeerId; + return { + result: { + fetchedDataTriples: found ? 1 : 0, + fetchedMetaTriples: found ? 8 : 0, + insertedTriples: found ? 9 : 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: found ? 'found' as const : 'clean-absent' as const, + }; + }; + (internals as any).reconcileChainOrdinal = async () => ( + lastPeerId === holderPeerId + ? { status: 'reconciled', blockNumber: 100 } + : { status: 'pending', recovery: target } + ); + + // Simulate a formerly authoritative prefix cached before the registry + // grew beyond the proof cap. The current oversized result must replace it + // with rotating transport windows instead of querying that prefix forever. + (internals as any).vmReconcileCuratorPeersByCg.set( + localCgId, + overflowPeers.slice(0, 4), + ); + + let result; + for (let pass = 0; pass < 5; pass += 1) { + result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + if (pass < 4) { + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + expect((internals as any).vmReconcileRotationState.get(slotKey)).toMatchObject({ + phase: 'collecting', curatorRosterConfirmed: false, + }); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + } + } + + expect(resolutions).toBe(5); + expect(fetches).toEqual(overflowPeers); + expect(result?.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + } finally { + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_EXACT_ROSTER_MAX', + rosterDescriptor, + ); + } + }); + + it('keeps a successful-empty metadata fallback ahead of the ordinary roster cap', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmFallbackCurator', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/fallback-curator'; + const fallbackPeer = '12D3KooWFallbackCuratorHolder'; + const ordinaryPeers = [1, 2, 3].map((n) => `12D3KooWFallbackOrdinary${n}`); + const connectedById = new Map( + ordinaryPeers.map((peerId) => [peerId, { toString: () => peerId }]), + ); + (internals as any).node = { + peerId: '12D3KooWFallbackCuratorLocal', + libp2p: { + getConnections: () => [...connectedById.values()].map((remotePeer) => ({ remotePeer })), + }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [], curatorIsLocal: false, legacyTripleResolved: false, lookupFailed: false, + }); + (internals as any).resolvePreferredSyncPeerId = async () => fallbackPeer; + (internals as any).ensurePeerConnected = async (peerId: string) => { + connectedById.set(peerId, { toString: () => peerId }); + }; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const fetches: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + fetches.push(peerId); + return { + result: { + fetchedDataTriples: peerId === fallbackPeer ? 1 : 0, + fetchedMetaTriples: peerId === fallbackPeer ? 8 : 0, + insertedTriples: peerId === fallbackPeer ? 9 : 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: peerId === fallbackPeer ? 'found' : 'clean-absent', + }; + }; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, 'fallback')], 100, () => true, + ); + + expect(fetches).toEqual([fallbackPeer]); + expect(result.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + expect((internals as any).vmReconcileCuratorPeersByCg.get(localCgId)) + .toEqual([fallbackPeer]); + }); + + it('clears cached authoritative curators after a successful empty resolution', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmClearCuratorCache', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/clear-curators'; + const stalePeer = '12D3KooWClearCuratorStale'; + const connectedPeer = { toString: () => stalePeer }; + (internals as any).node = { + peerId: '12D3KooWClearCuratorLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).vmReconcileCuratorPeersByCg.set(localCgId, [stalePeer]); + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [], curatorIsLocal: false, legacyTripleResolved: false, lookupFailed: false, + }); + (internals as any).resolvePreferredSyncPeerId = async () => undefined; + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = () => [connectedPeer]; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async () => ({ + result: { + fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'found', + }); + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + + await internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, '86')], 100, () => true, + ); + + expect((internals as any).vmReconcileCuratorPeersByCg.has(localCgId)).toBe(false); + }); + + it('retains cached authoritative curators when curator discovery fails', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRetainCuratorCache', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/retain-curators'; + const cachedPeer = '12D3KooWRetainCuratorCached'; + const connectedPeer = { toString: () => cachedPeer }; + (internals as any).node = { + peerId: '12D3KooWRetainCuratorLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).vmReconcileCuratorPeersByCg.set(localCgId, [cachedPeer]); + (internals as any).discovery.findAgents = async () => { + throw new Error('agent registry unavailable'); + }; + (internals as any).refreshMetaFromCurator = async () => false; + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = () => [connectedPeer]; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const fetches: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + fetches.push(peerId); + return { + result: { + fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'found', + }; + }; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + + await internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, '87')], 100, () => true, + ); + + expect(fetches).toEqual([cachedPeer]); + expect((internals as any).vmReconcileCuratorPeersByCg.get(localCgId)) + .toEqual([cachedPeer]); + }); + it('fetches one large recovery KA per peer attempt and defers the rest', async () => { const chain = new MockChainAdapter(); agent = await DKGAgent.create({ name: 'ExactVmBatchCap', chainAdapter: chain }); @@ -2118,15 +2655,18 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; (internals as any).waitForSyncProtocol = async () => true; const fetches: Array<{ peerId: string; uals: string[] }> = []; - (internals as any).syncExactKnowledgeAssetsFromPeer = async ( + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( peerId: string, _cg: string, uals: string[], ) => { fetches.push({ peerId, uals }); return { - fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, - failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'incomplete', }; }; const revalidated: number[] = []; @@ -2142,12 +2682,8 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { } return { status: 'reconciled', blockNumber: 100 }; }; - const targets = Array.from({ length: 4 }, (_, ordinal) => ({ - ordinal, - ual: `did:dkg:base:84532/0x0000000000000000000000000000000000000001/${ordinal}`, - kaId: String(ordinal), - reason: 'no-swm' as const, - })); + const targets = Array.from({ length: 4 }, (_, ordinal) => + vmRecoveryTarget(localCgId, ordinal)); const result = await internals.recoverVmReconcileBatch(localCgId, 1n, targets, 100, () => true); @@ -2165,6 +2701,47 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { expect(result.continuationOrdinal).toBe(2); }); + it('spreads unavailable-peer probes across targets within the global peer budget', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmUnavailableFairness', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peers = [1, 2, 3, 4].map((n) => `12D3KooWUnavailableFairness${n}`); + const localCgId = '0x0000000000000000000000000000000000000001/unavailable-fairness'; + (internals as any).node = { + peerId: '12D3KooWUnavailableFairnessLocal', + libp2p: { getConnections: () => [] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: peers, curatorIsLocal: false, legacyTripleResolved: false, + }); + const connectionAttempts: string[] = []; + (internals as any).ensurePeerConnected = async (peerId: string) => { + connectionAttempts.push(peerId); + }; + (internals as any).selectCatchupPeers = (candidates: Array<{ toString(): string }>) => candidates; + const protocolWaits: string[] = []; + (internals as any).waitForSyncProtocol = async (peerId: string) => { + protocolWaits.push(peerId); + return false; + }; + const fetch = vi.fn(); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + const targets = Array.from({ length: 4 }, (_, ordinal) => + vmRecoveryTarget(localCgId, ordinal, `unavailable-${ordinal}`)); + + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + + expect(connectionAttempts).toEqual(peers.slice(0, 3)); + // The dial stub deliberately leaves each peer disconnected, so protocol + // probing is skipped after each failed connection boundary. + expect(protocolWaits).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + expect(result.attemptedOrdinals).toEqual([0, 1, 2]); + expect(result.continuationOrdinal).toBe(3); + }); + it('tries each pending target once per eligible pass and damps immediate retries', async () => { const chain = new MockChainAdapter(); agent = await DKGAgent.create({ name: 'ExactVmBatchCooldown', chainAdapter: chain }); @@ -2185,29 +2762,22 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; (internals as any).waitForSyncProtocol = async () => true; const fetchedUals: string[][] = []; - (internals as any).syncExactKnowledgeAssetsFromPeer = async ( + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( _peerId: string, _cgId: string, uals: string[], ) => { fetchedUals.push(uals); return { - fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, - failedPeers: 1, failedPhases: 0, deferredBackpressure: 0, + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 1, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'incomplete', }; }; - const target = { - ordinal: 0, - ual: 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/7', - kaId: '7', - reason: 'no-swm' as const, - }; - const deferredTarget = { - ordinal: 1, - ual: 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/8', - kaId: '8', - reason: 'no-swm' as const, - }; + const target = vmRecoveryTarget(localCgId, 0, '7'); + const deferredTarget = vmRecoveryTarget(localCgId, 1, '8'); (internals as any).reconcileChainOrdinal = async ( _lcg: string, _ocg: bigint, @@ -2235,10 +2805,12 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { expect(second.cooldownOnly).toBe(true); }); - it('wraps recovery after the last pending target receives an eligible attempt', async () => { + it('suppresses completed incomplete cycles after every pending target receives an attempt', async () => { const chain = new MockChainAdapter(); agent = await DKGAgent.create({ name: 'ExactVmBatchWrap', chainAdapter: chain }); const internals = agent as unknown as AgentInternals; + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; const peer = '12D3KooWExactWrapPeer'; const localCgId = '0x0000000000000000000000000000000000000001/exact-wrap'; const connectedPeer = { toString: () => peer }; @@ -2256,21 +2828,19 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { (internals as any).selectCatchupPeers = () => [connectedPeer]; (internals as any).waitForSyncProtocol = async () => true; const networkAttempts: number[] = []; - const targets = [0, 1].map((ordinal) => ({ - ordinal, - ual: `did:dkg:base:84532/0x0000000000000000000000000000000000000001/${ordinal}`, - kaId: String(ordinal), - reason: 'no-swm' as const, - })); - (internals as any).syncExactKnowledgeAssetsFromPeer = async ( + const targets = [0, 1].map((ordinal) => vmRecoveryTarget(localCgId, ordinal)); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( _peerId: string, _cgId: string, uals: string[], ) => { networkAttempts.push(Number(uals[0]!.split('/').at(-1))); return { - fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, - failedPeers: 1, failedPhases: 0, deferredBackpressure: 0, + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 1, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'incomplete', }; }; (internals as any).reconcileChainOrdinal = async ( @@ -2295,12 +2865,19 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { expect(second.attemptedOrdinals).toEqual([1]); expect(second.continuationOrdinal).toBeUndefined(); + const rotationState = (internals as any).vmReconcileRotationState as Map; + expect(rotationState.size).toBe(2); + for (const target of targets) { + expect(rotationState.get( + (internals as any).vmReconcileRotationSlotKey(target), + )).toMatchObject({ phase: 'backoff', backoffKind: 'incomplete-cycle' }); + } (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); const wrapped = await internals.recoverVmReconcileBatch( localCgId, 1n, targets, 100, () => true, ); - expect(wrapped.attemptedOrdinals).toEqual([0]); - expect(networkAttempts).toEqual([0, 1, 0]); + expect(wrapped.attemptedOrdinals).toEqual([]); + expect(networkAttempts).toEqual([0, 1]); }); it('rotates one pending recovery target across eligible peers between windows', async () => { @@ -2329,24 +2906,22 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { (internals as any).ensurePeerAdmittedForRecovery = async () => true; const networkAttempts: string[] = []; let lastPeerId: string | undefined; - (internals as any).syncExactKnowledgeAssetsFromPeer = async (peerId: string) => { + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { networkAttempts.push(peerId); lastPeerId = peerId; return { - fetchedDataTriples: peerId === peerB ? 1 : 0, - fetchedMetaTriples: peerId === peerB ? 8 : 0, - insertedTriples: peerId === peerB ? 9 : 0, - failedPeers: 0, - failedPhases: 0, - deferredBackpressure: 0, + result: { + fetchedDataTriples: peerId === peerB ? 1 : 0, + fetchedMetaTriples: peerId === peerB ? 8 : 0, + insertedTriples: peerId === peerB ? 9 : 0, + failedPeers: 0, + failedPhases: 0, + deferredBackpressure: 0, + }, + disposition: peerId === peerB ? 'found' : 'clean-absent', }; }; - const target = { - ordinal: 0, - ual: 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/7', - kaId: '7', - reason: 'no-swm' as const, - }; + const target = vmRecoveryTarget(localCgId, 0, '7'); (internals as any).reconcileChainOrdinal = async () => ( lastPeerId === peerB ? { status: 'reconciled', blockNumber: 100 } @@ -2367,42 +2942,1661 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { expect(second.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); }); - it('clears the fetch cooldown after a productive exact batch so the next slice proceeds', async () => { + it('rotates incomplete, thrown, and still-pending found attempts without absence credit', async () => { const chain = new MockChainAdapter(); - agent = await DKGAgent.create({ name: 'ExactVmBatchProductive', chainAdapter: chain }); + agent = await DKGAgent.create({ name: 'ExactVmIncompleteRotation', chainAdapter: chain }); const internals = agent as unknown as AgentInternals; - const peer = '12D3KooWExactProductivePeer'; - const localCgId = '0x0000000000000000000000000000000000000001/exact-productive'; - const connected = [{ toString: () => peer }]; + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + const peerA = '12D3KooWIncompleteRotationPeerA'; + const peerB = '12D3KooWIncompleteRotationPeerB'; + const localCgId = '0x0000000000000000000000000000000000000001/incomplete-rotation'; + const connected = [peerA, peerB].map((peerId) => ({ toString: () => peerId })); (internals as any).node = { - peerId: '12D3KooWExactProductiveLocalPeer', + peerId: '12D3KooWIncompleteRotationLocal', libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, }; - (internals as any).preferredSyncPeers.set(localCgId, peer); + (internals as any).preferredSyncPeers.set(localCgId, peerA); (internals as any).resolveCuratorPeerIdsForCg = async () => ({ - peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + peerIds: [peerA], curatorIsLocal: false, legacyTripleResolved: false, }); (internals as any).ensurePeerConnected = async () => undefined; (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; (internals as any).waitForSyncProtocol = async () => true; - let fetchCount = 0; - (internals as any).syncExactKnowledgeAssetsFromPeer = async () => { - fetchCount += 1; - return { - fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, - failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const attemptsByUal = new Map(); + let lastDisposition: 'found' | 'incomplete' = 'incomplete'; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( + peerId: string, + _cgId: string, + requestedUals: string[], + ) => { + const ual = requestedUals[0]!; + const attempts = attemptsByUal.get(ual) ?? []; + attempts.push(peerId); + attemptsByUal.set(ual, attempts); + if (ual.endsWith('/77') && peerId === peerB) { + lastDisposition = 'incomplete'; + throw new Error('transport failed'); + } + lastDisposition = (ual.endsWith('/76') && peerId === peerB) + || (ual.endsWith('/77') && peerId === peerA) + ? 'found' + : 'incomplete'; + return { + result: { + fetchedDataTriples: lastDisposition === 'found' ? 1 : 0, + fetchedMetaTriples: lastDisposition === 'found' ? 8 : 0, + insertedTriples: lastDisposition === 'found' ? 9 : 0, + failedPeers: 0, failedPhases: lastDisposition === 'incomplete' ? 1 : 0, + deferredBackpressure: 0, + }, + disposition: lastDisposition, + }; + }; + let activeTarget = vmRecoveryTarget(localCgId, 0, '76'); + (internals as any).reconcileChainOrdinal = async () => ( + activeTarget.ual.endsWith('/76') && lastDisposition === 'found' + ? { status: 'reconciled', blockNumber: 100 } + : { status: 'pending', recovery: activeTarget } + ); + + await internals.recoverVmReconcileBatch(localCgId, 1n, [activeTarget], 100, () => true); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + const recovered = await internals.recoverVmReconcileBatch( + localCgId, 1n, [activeTarget], 100, () => true, + ); + expect(attemptsByUal.get(activeTarget.ual)).toEqual([peerA, peerB]); + expect(recovered.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + + activeTarget = vmRecoveryTarget(localCgId, 1, '77'); + lastDisposition = 'incomplete'; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [activeTarget], 100, () => true); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [activeTarget], 100, () => true); + expect(attemptsByUal.get(activeTarget.ual)).toEqual([peerA, peerB]); + const slotKey = (internals as any).vmReconcileRotationSlotKey(activeTarget); + const incompleteBackoff = (internals as any).vmReconcileRotationState.get(slotKey); + expect(incompleteBackoff).toMatchObject({ + phase: 'backoff', backoffKind: 'incomplete-cycle', failures: 1, + }); + expect([...incompleteBackoff.cleanAbsentPeerIds]).toEqual([]); + expect((internals as any).vmReconcileFetchCooldownAt.has(localCgId)).toBe(true); + + now = incompleteBackoff.nextRetryAt + 1; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + const retried = await internals.recoverVmReconcileBatch( + localCgId, 1n, [activeTarget], 100, () => true, + ); + expect(attemptsByUal.get(activeTarget.ual)).toEqual([peerA, peerB, peerA]); + expect(retried.outcomes.get(1)).toMatchObject({ status: 'pending' }); + expect((internals as any).vmReconcileRotationState.get(slotKey)).toMatchObject({ + phase: 'collecting', failures: 1, + }); + + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [activeTarget], 100, () => true); + expect(attemptsByUal.get(activeTarget.ual)).toEqual([peerA, peerB, peerA, peerB]); + expect((internals as any).vmReconcileRotationState.get(slotKey)).toMatchObject({ + phase: 'backoff', backoffKind: 'incomplete-cycle', failures: 2, + }); + }); + + it('backs off only after a complete clean-absence rotation and requires a fresh next cycle', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmCleanAbsenceBackoff', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peerA = '12D3KooWCleanAbsentPeerA'; + const peerB = '12D3KooWCleanAbsentPeerB'; + const localCgId = '0x0000000000000000000000000000000000000001/clean-absence'; + const connected = [peerA, peerB].map((peerId) => ({ toString: () => peerId })); + (internals as any).node = { + peerId: '12D3KooWCleanAbsentLocal', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peerA); + let curatorResolutions = 0; + (internals as any).resolveCuratorPeerIdsForCg = async () => { + curatorResolutions += 1; + return { peerIds: [peerA], curatorIsLocal: false, legacyTripleResolved: false }; + }; + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + const networkAttempts: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + networkAttempts.push(peerId); + return { + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'clean-absent', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, '71'); + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'pending', + recovery: target, + }); + + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + const firstBackoff = (internals as any).vmReconcileRotationState.get(slotKey); + expect(networkAttempts).toEqual([peerA, peerB]); + expect(firstBackoff).toMatchObject({ phase: 'backoff', failures: 1 }); + + const suppressed = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(suppressed.attemptedOrdinals).toEqual([]); + expect(curatorResolutions).toBe(2); + + now = firstBackoff.nextRetryAt + 1; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + const freshPartial = (internals as any).vmReconcileRotationState.get(slotKey); + expect(networkAttempts).toEqual([peerA, peerB, peerA]); + expect(freshPartial).toMatchObject({ phase: 'collecting', failures: 1 }); + expect([...freshPartial.cleanAbsentPeerIds]).toEqual([peerA]); + + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + expect(networkAttempts).toEqual([peerA, peerB, peerA, peerB]); + expect((internals as any).vmReconcileRotationState.get(slotKey)) + .toMatchObject({ phase: 'backoff', failures: 2 }); + }); + + it('backs off a scheduling-exhausted cycle when one peer is rejected and one is incomplete', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmSchedulingExhaustion', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const rejectedPeer = '12D3KooWSchedulingRejected'; + const incompletePeer = '12D3KooWSchedulingIncomplete'; + const localCgId = '0x0000000000000000000000000000000000000001/scheduling-exhaustion'; + const connected = [rejectedPeer, incompletePeer] + .map((peerId) => ({ toString: () => peerId })); + (internals as any).node = { + peerId: '12D3KooWSchedulingLocal', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).preferredSyncPeers.set(localCgId, rejectedPeer); + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [rejectedPeer], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async (peerId: string) => + peerId !== rejectedPeer; + const networkAttempts: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + networkAttempts.push(peerId); + return { + result: { + fetchedDataTriples: 50_000, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'incomplete', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, 'scheduling'); + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'pending', recovery: target, + }); + + const first = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(first.attemptedOrdinals).toEqual([0]); + expect(networkAttempts).toEqual([]); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + expect(networkAttempts).toEqual([incompletePeer]); + expect(result.attemptedOrdinals).toEqual([0]); + const incompleteBackoff = (internals as any).vmReconcileRotationState.get(slotKey); + expect(incompleteBackoff).toMatchObject({ + phase: 'backoff', backoffKind: 'incomplete-cycle', failures: 1, + }); + expect([...incompleteBackoff.attemptedPeerIds]) + .toEqual([rejectedPeer, incompletePeer]); + expect([...incompleteBackoff.cleanAbsentPeerIds]).toEqual([]); + expect((internals as any).vmReconcileFetchCooldownAt.has(localCgId)).toBe(true); + + const damped = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(damped.cooldownOnly).toBe(false); + expect(networkAttempts).toEqual([incompletePeer]); + }); + + it('retains max-batch proof progress while each slot continues making progress', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRollingCollectionDeadline', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const peers = ['12D3KooWRollingA', '12D3KooWRollingB', '12D3KooWRollingC']; + const targets = Array.from({ length: DKGAgent.VM_RECONCILE_BATCH_SIZE }, (_, ordinal) => + vmRecoveryTarget('rolling-collection-deadline', ordinal, `rolling-${ordinal}`)); + let now = 0; + (internals as any).vmReconcileRotationNow = () => now; + + for (const peerId of peers) { + for (const target of targets) { + now += 25_000; + const prepared = (internals as any).prepareVmReconcileRotationTarget( + target, peers, now, + ); + expect(prepared.suppressed).toBe(false); + expect(prepared.record).toBeDefined(); + (internals as any).settleVmReconcileRotationAttempt( + target, peerId, 'clean-absent', peers, prepared.record, + ); + } + } + + expect(now).toBeGreaterThan(DKGAgent.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS); + for (const target of targets) { + expect((internals as any).vmReconcileRotationState.get( + (internals as any).vmReconcileRotationSlotKey(target), + )).toMatchObject({ phase: 'backoff', failures: 1 }); + } + }); + + it('keeps the pre-suppression roster stable across connection reorder and curator discovery', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmCanonicalRoster', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peerA = '12D3KooWCanonicalRosterA'; + const peerB = '12D3KooWCanonicalRosterB'; + const peerC = '12D3KooWCanonicalRosterC'; + const peerD = '12D3KooWCanonicalRosterD'; + const localCgId = '0x0000000000000000000000000000000000000001/canonical-roster'; + const peerById = new Map( + [peerA, peerB, peerC, peerD].map((peerId) => [peerId, { toString: () => peerId }]), + ); + let connectionRead = 0; + (internals as any).node = { + peerId: '12D3KooWCanonicalRosterLocal', + libp2p: { + getConnections: () => { + connectionRead += 1; + const ids = connectionRead % 2 === 0 + ? [peerD, peerC, peerB, peerA] + : [peerB, peerD, peerA, peerC]; + return ids.map((peerId) => ({ remotePeer: peerById.get(peerId)! })); + }, + }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peerA); + let curatorResolutions = 0; + (internals as any).resolveCuratorPeerIdsForCg = async () => { + curatorResolutions += 1; + return { + // The secondary curator would displace C if this list were prepended to + // the post-discovery roster instead of using the canonicalizer. + peerIds: [peerA, peerD], + curatorIsLocal: false, + legacyTripleResolved: false, + }; + }; + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const networkAttempts: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + networkAttempts.push(peerId); + return { + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'clean-absent', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, '75'); + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'pending', recovery: target, + }); + + for (let attempt = 0; attempt < 3; attempt += 1) { + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + } + expect(networkAttempts).toEqual([peerA, peerD, peerB]); + expect(curatorResolutions).toBe(3); + + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + expect(networkAttempts).toEqual([peerA, peerD, peerB, peerC]); + expect(curatorResolutions).toBe(4); + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + expect((internals as any).vmReconcileRotationState.get(slotKey)).toMatchObject({ + phase: 'backoff', backoffKind: 'clean-absence', failures: 1, + }); + + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + expect(networkAttempts).toEqual([peerA, peerD, peerB, peerC]); + expect(curatorResolutions).toBe(4); + }); + + it('ranks a resolved structural curator ahead of an ordinary capped roster', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmResolvedCuratorRoster', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peers = [ + '12D3KooWResolvedCuratorA', + '12D3KooWResolvedCuratorB', + '12D3KooWResolvedCuratorC', + '12D3KooWResolvedCuratorD', + ]; + const curatorPeerId = peers[3]!; + const localCgId = '0x0000000000000000000000000000000000000001/resolved-curator'; + const connected = peers.map((peerId) => ({ toString: () => peerId })); + (internals as any).node = { + peerId: '12D3KooWResolvedCuratorLocal', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [curatorPeerId], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (ordered: Array<{ toString(): string }>) => ordered; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const attempts: string[] = []; + let lastPeerId: string | undefined; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + attempts.push(peerId); + lastPeerId = peerId; + return { + result: { + fetchedDataTriples: peerId === curatorPeerId ? 1 : 0, + fetchedMetaTriples: peerId === curatorPeerId ? 8 : 0, + insertedTriples: peerId === curatorPeerId ? 9 : 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: peerId === curatorPeerId ? 'found' : 'clean-absent', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, '80'); + (internals as any).reconcileChainOrdinal = async () => ( + lastPeerId === curatorPeerId + ? { status: 'reconciled', blockNumber: 100 } + : { status: 'pending', recovery: target } + ); + + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + + expect(attempts).toEqual([curatorPeerId]); + expect(result.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + }); + + it('completion-anchors the per-CG retry damper after a legacy incomplete rotation', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmPartialCycleExpiry', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peerA = '12D3KooWPartialCyclePeerA'; + const peerB = '12D3KooWPartialCyclePeerB'; + const localCgId = '0x0000000000000000000000000000000000000001/partial-cycle'; + const connected = [peerA, peerB].map((peerId) => ({ toString: () => peerId })); + (internals as any).node = { + peerId: '12D3KooWPartialCycleLocal', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peerA); + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peerA], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + const networkAttempts: string[] = []; + let peerAFetches = 0; + let lastDisposition: 'found' | 'clean-absent' | 'incomplete' = 'incomplete'; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + networkAttempts.push(peerId); + if (peerId === peerA) peerAFetches += 1; + lastDisposition = peerId === peerA && peerAFetches > 1 + ? 'found' + : peerId === peerA + ? 'clean-absent' + : 'incomplete'; + return { + result: { + fetchedDataTriples: lastDisposition === 'found' + ? 1 + : lastDisposition === 'incomplete' + ? 50_000 + : 0, + fetchedMetaTriples: lastDisposition === 'found' ? 8 : 0, + insertedTriples: lastDisposition === 'found' ? 9 : 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: lastDisposition, + }; + }; + const target = vmRecoveryTarget(localCgId, 0, '74'); + (internals as any).reconcileChainOrdinal = async () => ( + lastDisposition === 'found' + ? { status: 'reconciled', blockNumber: 100 } + : { status: 'pending', recovery: target } + ); + + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + expect(networkAttempts).toEqual([peerA, peerB]); + const incompleteBackoff = (internals as any).vmReconcileRotationState.get(slotKey); + expect(incompleteBackoff).toMatchObject({ + phase: 'backoff', backoffKind: 'incomplete-cycle', failures: 1, + }); + expect((internals as any).vmReconcileFetchCooldownAt.has(localCgId)).toBe(true); + + const damped = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(networkAttempts).toEqual([peerA, peerB]); + expect(damped.cooldownOnly).toBe(false); + + now = incompleteBackoff.nextRetryAt + 1; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + const recovered = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + + expect(networkAttempts).toEqual([peerA, peerB, peerA]); + expect(recovered.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + }); + + it('defers cap overflow until an expired slot can be replaced without retry churn', async () => { + const capDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_CACHE_MAX_ENTRIES', { + ...capDescriptor, + value: 1, + }); + try { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmStableRotationCapacity', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peerA = '12D3KooWStableCapacityPeerA'; + const peerB = '12D3KooWStableCapacityPeerB'; + const localCgId = '0x0000000000000000000000000000000000000001/stable-capacity'; + const connected = [peerA, peerB].map((peerId) => ({ toString: () => peerId })); + (internals as any).node = { + peerId: '12D3KooWStableCapacityLocal', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peerA); + let curatorResolutions = 0; + (internals as any).resolveCuratorPeerIdsForCg = async () => { + curatorResolutions += 1; + return { peerIds: [peerA], curatorIsLocal: false, legacyTripleResolved: false }; + }; + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const attemptsByUal = new Map(); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( + peerId: string, + _cgId: string, + requestedUals: string[], + ) => { + const ual = requestedUals[0]!; + const attempts = attemptsByUal.get(ual) ?? []; + attempts.push(peerId); + attemptsByUal.set(ual, attempts); + return { + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'clean-absent', + }; + }; + const first = vmRecoveryTarget(localCgId, 0, '78'); + const overflow = vmRecoveryTarget(localCgId, 1, '79'); + const secondOverflow = vmRecoveryTarget(localCgId, 2, '80'); + const targets = [first, overflow, secondOverflow]; + const byOrdinal = new Map(targets.map((target) => [target.ordinal, target])); + (internals as any).reconcileChainOrdinal = async ( + _lcg: string, + _ocg: bigint, + ordinal: number, + ) => ({ status: 'pending', recovery: byOrdinal.get(ordinal)! }); + const rotationState = (internals as any).vmReconcileRotationState as Map< + string, + { phase: string; cleanAbsentPeerIds: Set } + >; + const firstKey = (internals as any).vmReconcileRotationSlotKey(first); + const overflowKey = (internals as any).vmReconcileRotationSlotKey(overflow); + const secondOverflowKey = (internals as any).vmReconcileRotationSlotKey(secondOverflow); + + await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + expect(rotationState.size).toBe(1); + expect(rotationState.get(firstKey)).toMatchObject({ phase: 'collecting' }); + expect([...rotationState.get(firstKey)!.cleanAbsentPeerIds]).toEqual([peerA]); + expect(attemptsByUal.get(first.ual)).toEqual([peerA]); + expect(attemptsByUal.get(overflow.ual)).toBeUndefined(); + expect(attemptsByUal.get(secondOverflow.ual)).toBeUndefined(); + + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + expect(rotationState.size).toBe(1); + expect(rotationState.get(firstKey)).toMatchObject({ phase: 'backoff' }); + expect(attemptsByUal.get(first.ual)).toEqual([peerA, peerB]); + expect(attemptsByUal.get(overflow.ual)).toBeUndefined(); + + // A batch containing only an unowned target can be rejected from the + // live capacity state without paying curator discovery or transport. + const resolutionsBeforeCapacityDeferral = curatorResolutions; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch( + localCgId, 1n, [overflow], 100, () => true, + ); + expect(curatorResolutions).toBe(resolutionsBeforeCapacityDeferral); + expect(attemptsByUal.get(overflow.ual)).toBeUndefined(); + + // An unexpired backoff remains installed and the overflow target performs + // no evidence-free exact request. + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + expect(rotationState.size).toBe(1); + expect(rotationState.get(firstKey)).toMatchObject({ phase: 'backoff' }); + expect(rotationState.has(overflowKey)).toBe(false); + expect(attemptsByUal.get(first.ual)).toEqual([peerA, peerB]); + expect(attemptsByUal.get(overflow.ual)).toBeUndefined(); + + const firstBackoff = rotationState.get(firstKey) as any; + (internals as any).vmReconcileRotationNow = () => firstBackoff.nextRetryAt + 1; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + expect(rotationState.size).toBe(1); + expect(rotationState.has(firstKey)).toBe(false); + expect(rotationState.get(overflowKey)).toMatchObject({ phase: 'collecting' }); + expect(attemptsByUal.get(overflow.ual)).toEqual([peerA]); + + // Complete B's roster, expire it, and prove the third stable-order waiter + // receives the next slot instead of A/B alternating forever. + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + const overflowBackoff = rotationState.get(overflowKey) as any; + expect(overflowBackoff).toMatchObject({ phase: 'backoff' }); + expect(attemptsByUal.get(overflow.ual)).toEqual([peerA, peerB]); + + (internals as any).vmReconcileRotationNow = () => overflowBackoff.nextRetryAt + 1; + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + await internals.recoverVmReconcileBatch( + localCgId, 1n, targets, 100, () => true, + ); + expect(rotationState.size).toBe(1); + expect(rotationState.has(overflowKey)).toBe(false); + expect(rotationState.get(secondOverflowKey)).toMatchObject({ phase: 'collecting' }); + expect(attemptsByUal.get(secondOverflow.ual)).toEqual([peerA]); + } finally { + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + capDescriptor, + ); + } + }); + + it('reserves a live rotation slot for an unrelated CG at the node-wide cache cap', async () => { + const capDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_CACHE_MAX_ENTRIES', { + ...capDescriptor, + value: 3, + }); + try { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmCrossCgCapacity', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWCrossCgCapacityPeer'; + const dominantCg = '0x0000000000000000000000000000000000000001/dominant-capacity'; + const waitingCg = '0x0000000000000000000000000000000000000002/waiting-capacity'; + const dominantTargets = [0, 1, 2] + .map((ordinal) => vmRecoveryTarget(dominantCg, ordinal, `dominant-${ordinal}`)); + for (const target of dominantTargets) { + expect((internals as any).prepareVmReconcileRotationTarget( + target, + [peer], + 100, + ).suppressed).toBe(false); + } + const state = (internals as any).vmReconcileRotationState as Map< + string, + { localCgId: string } + >; + expect(state.size).toBe(3); + expect([...state.values()].filter((record) => record.localCgId === dominantCg)).toHaveLength(3); + + const waiting = vmRecoveryTarget(waitingCg, 0, 'waiting'); + const admitted = (internals as any).prepareVmReconcileRotationTarget( + waiting, + [peer], + 100, + ); + + expect(admitted.suppressed).toBe(false); + expect(admitted.record?.localCgId).toBe(waitingCg); + expect(state.size).toBe(3); + expect([...state.values()].filter((record) => record.localCgId === dominantCg)).toHaveLength(2); + expect([...state.values()].filter((record) => record.localCgId === waitingCg)).toHaveLength(1); + } finally { + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + capDescriptor, + ); + } + }); + + it('does not evict a cross-CG donor when recovery exits before requester installation', async () => { + const capDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_CACHE_MAX_ENTRIES', { + ...capDescriptor, + value: 3, + }); + try { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmDonationRollback', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWDonationRollbackPeer'; + const dominantCg = '0x0000000000000000000000000000000000000001/donation-owner'; + const waitingCg = '0x0000000000000000000000000000000000000002/donation-waiter'; + (internals as any).vmReconcileRotationNow = () => 100; + for (let ordinal = 0; ordinal < 3; ordinal += 1) { + const target = vmRecoveryTarget(dominantCg, ordinal, `donor-${ordinal}`); + expect((internals as any).prepareVmReconcileRotationTarget( + target, + [peer], + 100, + ).record).toBeDefined(); + } + const state = (internals as any).vmReconcileRotationState as Map; + const before = [...state.entries()]; + (internals as any).vmReconcileCuratorPeersByCg.set(waitingCg, [peer]); + let current = true; + (internals as any).resolveCuratorPeerIdsForCg = async () => { + current = false; + return { + peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + }; + }; + const waiting = vmRecoveryTarget(waitingCg, 0, 'waiting'); + + await internals.recoverVmReconcileBatch( + waitingCg, + 1n, + [waiting], + 100, + () => current, + ); + + expect([...state.entries()]).toEqual(before); + expect(state.has((internals as any).vmReconcileRotationSlotKey(waiting))).toBe(false); + } finally { + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + capDescriptor, + ); + } + }); + + it('retains an incomplete-cycle backoff until expiry, then releases its state slot', async () => { + const capDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_CACHE_MAX_ENTRIES', { + ...capDescriptor, + value: 1, + }); + try { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmNoProgressCapacity', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWNoProgressCapacityPeer'; + const first = vmRecoveryTarget('no-progress-capacity', 0, 'first'); + const second = vmRecoveryTarget('no-progress-capacity', 1, 'second'); + const firstKey = (internals as any).vmReconcileRotationSlotKey(first); + const secondKey = (internals as any).vmReconcileRotationSlotKey(second); + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + const firstRecord = (internals as any).prepareVmReconcileRotationTarget( + first, [peer], now, + ).record; + + (internals as any).settleVmReconcileRotationAttempt( + first, peer, 'incomplete', [peer], firstRecord, + ); + expect((internals as any).vmReconcileRotationState.get(firstKey)).toMatchObject({ + phase: 'backoff', backoffKind: 'incomplete-cycle', failures: 1, + }); + + const secondRecord = (internals as any).prepareVmReconcileRotationTarget( + second, [peer], now + 1, + ).record; + expect(secondRecord).toBeUndefined(); + expect((internals as any).vmReconcileRotationState.has(firstKey)).toBe(true); + expect((internals as any).vmReconcileRotationState.has(secondKey)).toBe(false); + + now = (internals as any).vmReconcileRotationState.get(firstKey).nextRetryAt + 1; + const admittedAfterExpiry = (internals as any).prepareVmReconcileRotationTarget( + second, [peer], now, + ).record; + expect(admittedAfterExpiry).toBeDefined(); + expect((internals as any).vmReconcileRotationState.has(firstKey)).toBe(false); + expect((internals as any).vmReconcileRotationState.get(secondKey)) + .toBe(admittedAfterExpiry); + } finally { + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + capDescriptor, + ); + } + }); + + it('evicts an expired collecting rotation when the next slot reaches the state cap', async () => { + const capDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_CACHE_MAX_ENTRIES', { + ...capDescriptor, + value: 1, + }); + try { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmExpiredCapacity', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWExpiredCapacityPeer'; + const first = vmRecoveryTarget('expired-capacity', 0, 'first'); + const second = vmRecoveryTarget('expired-capacity', 1, 'second'); + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + const firstKey = (internals as any).vmReconcileRotationSlotKey(first); + const secondKey = (internals as any).vmReconcileRotationSlotKey(second); + expect((internals as any).prepareVmReconcileRotationTarget( + first, [peer], now, + ).record).toBeDefined(); + + now += DKGAgent.VM_RECONCILE_NEGATIVE_BACKOFF_MAX_MS + 1; + const secondRecord = (internals as any).prepareVmReconcileRotationTarget( + second, [peer], now, + ).record; + expect(secondRecord).toBeDefined(); + expect((internals as any).vmReconcileRotationState.has(firstKey)).toBe(false); + expect((internals as any).vmReconcileRotationState.get(secondKey)).toBe(secondRecord); + } finally { + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_CACHE_MAX_ENTRIES', + capDescriptor, + ); + } + }); + + it('preserves retained proof across candidate growth and ignores evicted records', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRotationIdentity', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const target = vmRecoveryTarget('rotation-identity', 0, '72'); + const peerA = '12D3KooWRotationIdentityA'; + const peerB = '12D3KooWRotationIdentityB'; + const peerC = '12D3KooWRotationIdentityC'; + + const initial = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA, peerB], 0, + ); + (internals as any).creditVmReconcileCleanAbsence( + target, peerA, [peerA, peerB], initial.record, + ); + expect([...initial.record.cleanAbsentPeerIds]).toEqual([peerA]); + + const grown = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA, peerB, peerC], 1, + ); + expect(grown.record).toBe(initial.record); + expect(grown.record.phase).toBe('collecting'); + expect([...grown.record.cleanAbsentPeerIds]).toEqual([peerA]); + expect([...grown.record.attemptedPeerIds]).toEqual([peerA]); + + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + (internals as any).vmReconcileRotationState.delete(slotKey); + (internals as any).creditVmReconcileCleanAbsence( + target, peerA, [peerA, peerB, peerC], grown.record, + ); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + + const rootB = { ...target, merkleRoot: 'root-b' }; + const replacement = (internals as any).prepareVmReconcileRotationTarget( + rootB, [peerA], 2, + ).record; + expect(replacement).not.toBe(grown.record); + const rootAAgain = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA], 2, + ).record; + expect(rootAAgain).not.toBe(initial.record); + expect(rootAAgain).not.toBe(replacement); + (internals as any).forceClearVmReconcileStateForContextGraph(target.localCgId); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + (internals as any).creditVmReconcileCleanAbsence( + target, peerA, [peerA], rootAAgain, + ); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + + const shutdownRecord = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA], 3, + ).record; + (internals as any).closeVmReconcileRotationState(); + (internals as any).creditVmReconcileCleanAbsence( + target, peerA, [peerA], shutdownRecord, + ); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it('closes exact-recovery rotation state through the public stop path', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRotationPublicStop', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const target = vmRecoveryTarget('rotation-public-stop', 0, 'stop'); + const peer = '12D3KooWRotationPublicStop'; + (internals as any).started = true; + (internals as any).node = { + peerId: '12D3KooWRotationPublicStopLocal', + libp2p: { getPeers: () => [] }, + stop: vi.fn(async () => undefined), + }; + (internals as any).messenger = { stopOutboxDrain: vi.fn(async () => undefined) }; + const record = (internals as any).prepareVmReconcileRotationTarget( + target, [peer], 100, + ).record; + expect((internals as any).vmReconcileRotationState.size).toBe(1); + + await agent.stop(); + + expect((internals as any).vmReconcileRotationClosed).toBe(true); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + (internals as any).settleVmReconcileRotationAttempt( + target, peer, 'clean-absent', [peer], record, + ); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + agent = null; + }); + + it('clears the complete proof cycle across capped-roster replacement and rejoin', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRotationReplacement', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const target = vmRecoveryTarget('rotation-replacement', 0, '81'); + const peerA = '12D3KooWRotationReplacementA'; + const peerB = '12D3KooWRotationReplacementB'; + const peerC = '12D3KooWRotationReplacementC'; + const peerD = '12D3KooWRotationReplacementD'; + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + + const initial = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA, peerB, peerC], now, + ).record; + for (const peerId of [peerA, peerB, peerC]) { + (internals as any).settleVmReconcileRotationAttempt( + target, peerId, 'clean-absent', [peerA, peerB, peerC], initial, + ); + } + expect(initial).toMatchObject({ phase: 'backoff', failures: 1 }); + + now += 1; + const replaced = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA, peerC, peerD], now, + ).record; + expect(replaced).toBe(initial); + expect(replaced).toMatchObject({ phase: 'collecting', failures: 1, nextRetryAt: 0 }); + expect([...replaced.attemptedPeerIds]).toEqual([]); + expect([...replaced.cleanAbsentPeerIds]).toEqual([]); + expect((internals as any).vmReconcileUncreditedCandidateOrder(replaced)) + .toEqual([peerA, peerC, peerD]); + (internals as any).settleVmReconcileRotationAttempt( + target, peerA, 'clean-absent', [peerA, peerC, peerD], replaced, + ); + expect([...replaced.cleanAbsentPeerIds]).toEqual([peerA]); + + now += 1; + const rejoined = (internals as any).prepareVmReconcileRotationTarget( + target, [peerA, peerB, peerC], now, + ).record; + expect(rejoined).toBe(initial); + expect(rejoined).toMatchObject({ phase: 'collecting', failures: 1, nextRetryAt: 0 }); + expect([...rejoined.attemptedPeerIds]).toEqual([]); + expect([...rejoined.cleanAbsentPeerIds]).toEqual([]); + expect((internals as any).vmReconcileUncreditedCandidateOrder(rejoined)) + .toEqual([peerA, peerB, peerC]); + }); + + it('clears partial and active-backoff evidence on roster shrink', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRotationShrink', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const peerA = '12D3KooWRotationShrinkA'; + const peerB = '12D3KooWRotationShrinkB'; + + const partialTarget = vmRecoveryTarget('rotation-shrink', 0, '87'); + const partial = (internals as any).prepareVmReconcileRotationTarget( + partialTarget, [peerA, peerB], 100, + ).record; + (internals as any).settleVmReconcileRotationAttempt( + partialTarget, peerA, 'incomplete', [peerA, peerB], partial, + ); + expect([...partial.attemptedPeerIds]).toEqual([peerA]); + + const shrunkPartial = (internals as any).prepareVmReconcileRotationTarget( + partialTarget, [peerA], 101, + ); + expect(shrunkPartial.suppressed).toBe(false); + expect(shrunkPartial.record).toMatchObject({ phase: 'collecting', failures: 0 }); + expect([...shrunkPartial.record.attemptedPeerIds]).toEqual([]); + expect([...shrunkPartial.record.cleanAbsentPeerIds]).toEqual([]); + // A response captured against the old roster is inert after shrink. + (internals as any).settleVmReconcileRotationAttempt( + partialTarget, peerB, 'clean-absent', [peerA, peerB], partial, + ); + expect([...shrunkPartial.record.cleanAbsentPeerIds]).toEqual([]); + + const backoffTarget = vmRecoveryTarget('rotation-shrink', 1, '88'); + const backoff = (internals as any).prepareVmReconcileRotationTarget( + backoffTarget, [peerA, peerB], 200, + ).record; + for (const peerId of [peerA, peerB]) { + (internals as any).settleVmReconcileRotationAttempt( + backoffTarget, peerId, 'clean-absent', [peerA, peerB], backoff, + ); + } + expect(backoff.phase).toBe('backoff'); + const shrunkBackoff = (internals as any).prepareVmReconcileRotationTarget( + backoffTarget, [peerA], 201, + ); + expect(shrunkBackoff.suppressed).toBe(false); + expect(shrunkBackoff.record).toMatchObject({ phase: 'collecting', failures: 1 }); + expect([...shrunkBackoff.record.attemptedPeerIds]).toEqual([]); + expect([...shrunkBackoff.record.cleanAbsentPeerIds]).toEqual([]); + }); + + it('preserves active backoff across an empty socket view but drops partial evidence', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmEmptyRoster', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWEmptyRosterPeer'; + const otherPeer = '12D3KooWEmptyRosterOther'; + const localCgId = '0x0000000000000000000000000000000000000001/empty-roster'; + (internals as any).node = { + peerId: '12D3KooWEmptyRosterLocal', + libp2p: { getConnections: () => [] }, + }; + let now = 100; + (internals as any).vmReconcileRotationNow = () => now; + + const partialTarget = vmRecoveryTarget(localCgId, 0, 'partial-empty'); + const partial = (internals as any).prepareVmReconcileRotationTarget( + partialTarget, [peer, otherPeer], now, + ).record; + (internals as any).settleVmReconcileRotationAttempt( + partialTarget, peer, 'incomplete', [peer, otherPeer], partial, + ); + expect([...partial.attemptedPeerIds]).toEqual([peer]); + const partialKey = (internals as any).vmReconcileRotationSlotKey(partialTarget); + const emptyPartial = (internals as any).prepareVmReconcileRotationTarget( + partialTarget, [], now + 1, + ); + expect(emptyPartial.suppressed).toBe(false); + expect((internals as any).vmReconcileRotationState.has(partialKey)).toBe(false); + const rejoinedPartial = (internals as any).prepareVmReconcileRotationTarget( + partialTarget, [peer], now + 2, + ).record; + expect([...rejoinedPartial.attemptedPeerIds]).toEqual([]); + expect([...rejoinedPartial.cleanAbsentPeerIds]).toEqual([]); + + const backoffTarget = vmRecoveryTarget(localCgId, 1, 'backoff-empty'); + const backoff = (internals as any).prepareVmReconcileRotationTarget( + backoffTarget, [peer], now, + ).record; + (internals as any).settleVmReconcileRotationAttempt( + backoffTarget, peer, 'clean-absent', [peer], backoff, + ); + expect(backoff.phase).toBe('backoff'); + now += 1; + + const resolveCurators = vi.fn(); + (internals as any).resolveCuratorPeerIdsForCg = resolveCurators; + const fetch = vi.fn(); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [backoffTarget], 100, () => true, + ); + expect(result.attemptedOrdinals).toEqual([]); + expect(resolveCurators).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + expect((internals as any).vmReconcileRotationState.get( + (internals as any).vmReconcileRotationSlotKey(backoffTarget), + )).toBe(backoff); + }); + + it('does not let an unconfirmed curator roster suppress the next discovery attempt', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmUnconfirmedCuratorBackoff', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/unconfirmed-curator'; + const ordinaryPeer = '12D3KooWUnconfirmedOrdinary'; + const target = vmRecoveryTarget(localCgId, 0, 'unconfirmed'); + const first = (internals as any).prepareVmReconcileRotationTarget( + target, [ordinaryPeer], 100, false, + ); + (internals as any).settleVmReconcileRotationAttempt( + target, ordinaryPeer, 'clean-absent', [ordinaryPeer], first.record, + ); + expect(first.record).toMatchObject({ + phase: 'collecting', + curatorRosterConfirmed: false, + }); + + const retry = (internals as any).prepareVmReconcileRotationTarget( + target, [ordinaryPeer], 101, false, + ); + expect(retry.suppressed).toBe(false); + expect(retry.record).toBe(first.record); + expect(retry.record).toMatchObject({ + phase: 'collecting', + curatorRosterConfirmed: false, + }); + + (internals as any).settleVmReconcileRotationAttempt( + target, ordinaryPeer, 'clean-absent', [ordinaryPeer], retry.record, + ); + const confirmed = (internals as any).prepareVmReconcileRotationTarget( + target, [ordinaryPeer], 102, true, + ); + expect(confirmed.suppressed).toBe(false); + expect(confirmed.record).toBe(retry.record); + expect(confirmed.record.curatorRosterConfirmed).toBe(true); + expect([...confirmed.record.attemptedPeerIds]).toEqual([]); + (internals as any).settleVmReconcileRotationAttempt( + target, ordinaryPeer, 'clean-absent', [ordinaryPeer], confirmed.record, + ); + expect((internals as any).prepareVmReconcileRotationTarget( + target, [ordinaryPeer], 103, true, + ).suppressed).toBe(true); + }); + + it('reprobes retained peers when curator proof arrives with roster growth', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmCuratorProofGrowth', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/proof-growth'; + const ordinaryPeer = '12D3KooWProofGrowthOrdinary'; + const curatorPeer = '12D3KooWProofGrowthCurator'; + const target = vmRecoveryTarget(localCgId, 0, 'proof-growth'); + const unconfirmed = (internals as any).prepareVmReconcileRotationTarget( + target, [ordinaryPeer], 100, false, + ); + (internals as any).settleVmReconcileRotationAttempt( + target, ordinaryPeer, 'clean-absent', [ordinaryPeer], unconfirmed.record, + ); + expect([...unconfirmed.record.cleanAbsentPeerIds]).toEqual([ordinaryPeer]); + + const confirmed = (internals as any).prepareVmReconcileRotationTarget( + target, [ordinaryPeer, curatorPeer], 101, true, + ); + expect(confirmed.suppressed).toBe(false); + expect(confirmed.record).toBe(unconfirmed.record); + expect(confirmed.record.curatorRosterConfirmed).toBe(true); + expect([...confirmed.record.candidatePeerIds]).toEqual([ordinaryPeer, curatorPeer]); + expect([...confirmed.record.attemptedPeerIds]).toEqual([]); + expect([...confirmed.record.cleanAbsentPeerIds]).toEqual([]); + }); + + it('does not persist incomplete-cycle suppression when curator discovery is unconfirmed', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmUnconfirmedIncompleteBackoff', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/unconfirmed-incomplete'; + const peer = '12D3KooWUnconfirmedIncomplete'; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWUnconfirmedIncompleteLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peer); + let curatorResolutions = 0; + (internals as any).resolveCuratorPeerIdsForCg = async () => { + curatorResolutions += 1; + return { + peerIds: [], curatorIsLocal: false, legacyTripleResolved: false, lookupFailed: true, + }; + }; + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = () => [connectedPeer]; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + let fetches = 0; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async () => { + fetches += 1; + return { + result: { + fetchedDataTriples: 50_000, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'incomplete', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, 'unconfirmed-incomplete'); + (internals as any).reconcileChainOrdinal = async () => ({ status: 'pending', recovery: target }); + + await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + expect((internals as any).vmReconcileRotationState.get(slotKey)).toMatchObject({ + phase: 'collecting', curatorRosterConfirmed: false, + }); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + const suppressed = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + + expect(suppressed.attemptedOrdinals).toEqual([]); + expect(curatorResolutions).toBe(2); + expect(fetches).toBe(1); + }); + + it('retries curator discovery after an unconfirmed clean-absence cycle', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmCuratorDiscoveryRetry', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/curator-retry'; + const ordinaryPeer = '12D3KooWCuratorRetryOrdinary'; + const curatorPeer = '12D3KooWCuratorRetryAuthoritative'; + const connectedById = new Map([ + [ordinaryPeer, { toString: () => ordinaryPeer }], + ]); + (internals as any).node = { + peerId: '12D3KooWCuratorRetryLocal', + libp2p: { + getConnections: () => [...connectedById.values()] + .map((remotePeer) => ({ remotePeer })), + }, + }; + const resolveCurators = vi.fn() + .mockResolvedValueOnce({ + peerIds: [], curatorIsLocal: false, legacyTripleResolved: false, lookupFailed: true, + }) + .mockResolvedValueOnce({ + peerIds: [curatorPeer], curatorIsLocal: false, + legacyTripleResolved: false, lookupFailed: false, + }); + (internals as any).resolveCuratorPeerIdsForCg = resolveCurators; + (internals as any).ensurePeerConnected = async (peerId: string) => { + connectedById.set(peerId, { toString: () => peerId }); + }; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + const fetches: string[] = []; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { + fetches.push(peerId); + const found = peerId === curatorPeer; + return { + result: { + fetchedDataTriples: found ? 1 : 0, + fetchedMetaTriples: found ? 8 : 0, + insertedTriples: found ? 9 : 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: found ? 'found' : 'clean-absent', + }; + }; + (internals as any).reconcileChainOrdinal = vi.fn() + .mockResolvedValueOnce({ status: 'pending' }) + .mockResolvedValue({ status: 'reconciled', blockNumber: 100 }); + const target = vmRecoveryTarget(localCgId, 0, 'curator-retry'); + + const first = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(first.outcomes.get(0)).toEqual({ status: 'pending' }); + expect(fetches).toEqual([ordinaryPeer]); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + + const second = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(resolveCurators).toHaveBeenCalledTimes(2); + expect(fetches).toEqual([ordinaryPeer, curatorPeer]); + expect(second.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); + }); + + it('ignores stale exact-recovery targets before creating state or fetching', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmStaleTarget', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const fetch = vi.fn(); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + const staleLocal = vmRecoveryTarget('other-context-graph', 0, '82'); + const staleOnChain = { + ...vmRecoveryTarget('current-context-graph', 1, '83'), + onChainCgId: '2', + }; + + const result = await internals.recoverVmReconcileBatch( + 'current-context-graph', 1n, [staleLocal, staleOnChain], 100, () => true, + ); + + expect(result.attemptedOrdinals).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it.each(['unsubscribe', 'rebind'])('does not recreate state after %s during curator resolution', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmResolutionInvalidation', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/resolution-invalidation'; + const peer = '12D3KooWResolutionInvalidationPeer'; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWResolutionInvalidationLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + let releaseResolution!: () => void; + let markResolutionStarted!: () => void; + const resolutionStarted = new Promise((resolve) => { markResolutionStarted = resolve; }); + const resolutionRelease = new Promise((resolve) => { releaseResolution = resolve; }); + (internals as any).resolveCuratorPeerIdsForCg = async () => { + markResolutionStarted(); + await resolutionRelease; + return { peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false }; + }; + const connect = recorder(async () => undefined); + const fetch = recorder(async () => undefined); + (internals as any).ensurePeerConnected = connect; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + let current = true; + const target = vmRecoveryTarget(localCgId, 0, 'resolution-invalidation'); + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => current, + ); + await resolutionStarted; + current = false; + (internals as any).forceClearVmReconcileStateForContextGraph(localCgId); + releaseResolution(); + + await expect(recovery).resolves.toMatchObject({ attemptedOrdinals: [] }); + expect(connect.calls).toEqual([]); + expect(fetch.calls).toEqual([]); + expect((internals as any).vmReconcileCuratorPeersByCg.has(localCgId)).toBe(false); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it('does not recreate state after shutdown during fallback resolution', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmFallbackShutdown', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/fallback-shutdown'; + const peer = '12D3KooWFallbackShutdownPeer'; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWFallbackShutdownLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [], curatorIsLocal: false, legacyTripleResolved: false, + }); + let releaseFallback!: () => void; + let markFallbackStarted!: () => void; + const fallbackStarted = new Promise((resolve) => { markFallbackStarted = resolve; }); + const fallbackRelease = new Promise((resolve) => { releaseFallback = resolve; }); + (internals as any).resolvePreferredSyncPeerId = async () => { + markFallbackStarted(); + await fallbackRelease; + return peer; + }; + const connect = recorder(async () => undefined); + (internals as any).ensurePeerConnected = connect; + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, 'fallback-shutdown')], 100, () => true, + ); + await fallbackStarted; + (internals as any).closeVmReconcileRotationState(); + releaseFallback(); + + await expect(recovery).resolves.toMatchObject({ attemptedOrdinals: [] }); + expect(connect.calls).toEqual([]); + expect((internals as any).vmReconcileCuratorPeersByCg.size).toBe(0); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it('keeps a pre-stop exact recovery stale after rotation state reopens', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmRestartGeneration', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/restart-generation'; + const peer = '12D3KooWRestartGenerationPeer'; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWRestartGenerationLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [], curatorIsLocal: false, legacyTripleResolved: false, + }); + let releaseFallback!: () => void; + let markFallbackStarted!: () => void; + const fallbackStarted = new Promise((resolve) => { markFallbackStarted = resolve; }); + const fallbackRelease = new Promise((resolve) => { releaseFallback = resolve; }); + (internals as any).resolvePreferredSyncPeerId = async () => { + markFallbackStarted(); + await fallbackRelease; + return peer; + }; + const connect = recorder(async () => undefined); + const fetch = recorder(async () => undefined); + (internals as any).ensurePeerConnected = connect; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, 'restart-generation')], 100, () => true, + ); + await fallbackStarted; + const priorGeneration = (internals as any).vmReconcileLifecycleGeneration; + (internals as any).closeVmReconcileRotationState(); + (internals as any).openVmReconcileRotationState(); + expect((internals as any).vmReconcileRotationClosed).toBe(false); + expect((internals as any).vmReconcileLifecycleGeneration).toBe(priorGeneration + 1); + releaseFallback(); + + await expect(recovery).resolves.toMatchObject({ attemptedOrdinals: [] }); + expect(connect.calls).toEqual([]); + expect(fetch.calls).toEqual([]); + expect((internals as any).vmReconcileCuratorPeersByCg.size).toBe(0); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it('does not recreate state after a rebind while dialing the curator', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmDialInvalidation', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/dial-invalidation'; + const peer = '12D3KooWDialInvalidationPeer'; + (internals as any).node = { + peerId: '12D3KooWDialInvalidationLocal', + libp2p: { getConnections: () => [] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + }); + let releaseDial!: () => void; + let markDialStarted!: () => void; + const dialStarted = new Promise((resolve) => { markDialStarted = resolve; }); + const dialRelease = new Promise((resolve) => { releaseDial = resolve; }); + (internals as any).ensurePeerConnected = async () => { + markDialStarted(); + await dialRelease; + }; + const fetch = recorder(async () => undefined); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + let current = true; + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [vmRecoveryTarget(localCgId, 0, 'dial-invalidation')], 100, () => current, + ); + await dialStarted; + current = false; + (internals as any).forceClearVmReconcileStateForContextGraph(localCgId); + releaseDial(); + + await expect(recovery).resolves.toMatchObject({ attemptedOrdinals: [] }); + expect(fetch.calls).toEqual([]); + expect((internals as any).vmReconcileCuratorPeersByCg.has(localCgId)).toBe(false); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it.each(['protocol', 'admission'] as const)( + 'does not start stale transport after invalidation during the %s wait', + async (stage) => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmBoundaryInvalidation', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = `0x0000000000000000000000000000000000000001/${stage}-invalidation`; + const peer = `12D3KooW${stage}InvalidationPeer`; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWBoundaryInvalidationLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + let releaseBoundary!: () => void; + let markBoundaryStarted!: () => void; + const boundaryStarted = new Promise((resolve) => { markBoundaryStarted = resolve; }); + const boundaryRelease = new Promise((resolve) => { releaseBoundary = resolve; }); + (internals as any).waitForSyncProtocol = async () => { + if (stage === 'protocol') { + markBoundaryStarted(); + await boundaryRelease; + } + return true; + }; + (internals as any).ensurePeerAdmittedForRecovery = async () => { + if (stage === 'admission') { + markBoundaryStarted(); + await boundaryRelease; + } + return true; + }; + const fetch = vi.fn(); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = fetch; + let current = true; + const target = vmRecoveryTarget(localCgId, 0, `${stage}-invalidation`); + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => current, + ); + await boundaryStarted; + current = false; + (internals as any).forceClearVmReconcileStateForContextGraph(localCgId); + releaseBoundary(); + + await expect(recovery).resolves.toMatchObject({ attemptedOrdinals: [] }); + expect(fetch).not.toHaveBeenCalled(); + expect((internals as any).vmReconcileFetchCooldownAt.has(localCgId)).toBe(false); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }, + ); + + it('does not restore cooldown after invalidation during exact transport', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmTransportInvalidation', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const localCgId = '0x0000000000000000000000000000000000000001/transport-invalidation'; + const peer = '12D3KooWTransportInvalidationPeer'; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWTransportInvalidationLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + let releaseFetch!: () => void; + let markFetchStarted!: () => void; + const fetchStarted = new Promise((resolve) => { markFetchStarted = resolve; }); + const fetchRelease = new Promise((resolve) => { releaseFetch = resolve; }); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async () => { + markFetchStarted(); + await fetchRelease; + return { + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'clean-absent', + }; + }; + const reconcile = vi.fn(); + (internals as any).reconcileChainOrdinal = reconcile; + let current = true; + const target = vmRecoveryTarget(localCgId, 0, 'transport-invalidation'); + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => current, + ); + await fetchStarted; + current = false; + (internals as any).forceClearVmReconcileStateForContextGraph(localCgId); + releaseFetch(); + + await expect(recovery).resolves.toMatchObject({ attemptedOrdinals: [] }); + expect(reconcile).not.toHaveBeenCalled(); + expect((internals as any).vmReconcileFetchCooldownAt.has(localCgId)).toBe(false); + expect((internals as any).vmReconcileRotationState.size).toBe(0); + }); + + it('does not resurrect a rotation record evicted while the exact request is pending', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmLateEviction', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWLateEvictionPeer'; + const localCgId = '0x0000000000000000000000000000000000000001/late-eviction'; + const connectedPeer = { toString: () => peer }; + (internals as any).node = { + peerId: '12D3KooWLateEvictionLocal', + libp2p: { getConnections: () => [{ remotePeer: connectedPeer }] }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peer); + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = () => [connectedPeer]; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + let releaseFetch!: () => void; + let markFetchStarted!: () => void; + const fetchStarted = new Promise((resolve) => { markFetchStarted = resolve; }); + const fetchRelease = new Promise((resolve) => { releaseFetch = resolve; }); + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async () => { + markFetchStarted(); + await fetchRelease; + return { + result: { + fetchedDataTriples: 0, fetchedMetaTriples: 0, insertedTriples: 0, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'clean-absent', + }; + }; + const target = vmRecoveryTarget(localCgId, 0, '73'); + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'pending', recovery: target, + }); + + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + await fetchStarted; + const slotKey = (internals as any).vmReconcileRotationSlotKey(target); + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(true); + (internals as any).vmReconcileRotationState.delete(slotKey); + releaseFetch(); + await recovery; + + expect((internals as any).vmReconcileRotationState.has(slotKey)).toBe(false); + }); + + it('clears the fetch cooldown after a productive exact batch so the next slice proceeds', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmBatchProductive', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peer = '12D3KooWExactProductivePeer'; + const localCgId = '0x0000000000000000000000000000000000000001/exact-productive'; + const connected = [{ toString: () => peer }]; + (internals as any).node = { + peerId: '12D3KooWExactProductiveLocalPeer', + libp2p: { getConnections: () => connected.map((remotePeer) => ({ remotePeer })) }, + }; + (internals as any).preferredSyncPeers.set(localCgId, peer); + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peer], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).ensurePeerConnected = async () => undefined; + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + let fetchCount = 0; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async () => { + fetchCount += 1; + return { + result: { + fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'found', }; }; (internals as any).reconcileChainOrdinal = async () => ({ status: 'reconciled', blockNumber: 100, }); - const target = { - ordinal: 0, - ual: 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/7', - kaId: '7', - reason: 'no-swm' as const, - }; + const target = vmRecoveryTarget(localCgId, 0, '7'); await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); expect(fetchCount).toBe(1); @@ -2439,31 +4633,348 @@ describe('Phase D — reconcile gate + core-fill telemetry', () => { ensureAdmitted: async () => false, }; const fetches: string[] = []; - (internals as any).syncExactKnowledgeAssetsFromPeer = async (peerId: string) => { + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async (peerId: string) => { fetches.push(peerId); return { - fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, - failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + result: { + fetchedDataTriples: 1, fetchedMetaTriples: 8, insertedTriples: 9, + failedPeers: 0, failedPhases: 0, deferredBackpressure: 0, + }, + disposition: 'found', }; }; (internals as any).reconcileChainOrdinal = async () => ({ status: 'reconciled', blockNumber: 100, }); - const target = { - ordinal: 0, - ual: 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/7', - kaId: '7', - reason: 'no-swm' as const, - }; + const target = vmRecoveryTarget(localCgId, 0, '7'); - const result = await internals.recoverVmReconcileBatch(localCgId, 1n, [target], 100, () => true); + const first = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); + expect(first.attemptedOrdinals).toEqual([0]); + expect(fetches).toEqual([]); + (internals as any).vmReconcileFetchCooldownAt.delete(localCgId); + const result = await internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, + ); expect(fetches).toEqual([admittedPeer]); expect(result.outcomes.get(0)).toEqual({ status: 'reconciled', blockNumber: 100 }); expect(result.attemptedOrdinals).toEqual([0]); }); + it('propagates lifecycle cancellation into an in-flight exact recovery', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'ExactVmLifecycleAbort', chainAdapter: chain }); + const internals = agent as unknown as AgentInternals; + const peerId = '12D3KooWExactLifecycleAbortPeer'; + const localCgId = '0x0000000000000000000000000000000000000001/exact-abort'; + const remotePeer = { toString: () => peerId }; + (internals as any).node = { + peerId: '12D3KooWExactLifecycleAbortLocal', + libp2p: { getConnections: () => [{ remotePeer }] }, + }; + (internals as any).resolveCuratorPeerIdsForCg = async () => ({ + peerIds: [peerId], curatorIsLocal: false, legacyTripleResolved: false, + }); + (internals as any).selectCatchupPeers = (peers: Array<{ toString(): string }>) => peers; + (internals as any).waitForSyncProtocol = async () => true; + (internals as any).ensurePeerAdmittedForRecovery = async () => true; + let markEntered!: () => void; + const entered = new Promise((resolve) => { markEntered = resolve; }); + let receivedSignal: AbortSignal | undefined; + (internals as any).syncExactKnowledgeAssetsFromPeerDetailed = async ( + _peerId: string, + _cgId: string, + _uals: string[], + options: { signal?: AbortSignal }, + ) => { + receivedSignal = options.signal; + markEntered(); + await new Promise((_resolve, reject) => { + const onAbort = () => reject(new DOMException('aborted', 'AbortError')); + if (options.signal?.aborted) onAbort(); + else options.signal?.addEventListener('abort', onAbort, { once: true }); + }); + throw new Error('unreachable'); + }; + const target = vmRecoveryTarget(localCgId, 0, 'exact-abort'); + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'pending', recovery: target, + }); + const controller = new AbortController(); + + const recovery = internals.recoverVmReconcileBatch( + localCgId, 1n, [target], 100, () => true, controller.signal, + ); + await entered; + controller.abort(); + + await expect(recovery).resolves.toMatchObject({ outcomes: new Map() }); + expect(receivedSignal).toBe(controller.signal); + }); + + it('abandons a same-object context-graph rebind that lands during stranded-KC repair', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillHealBindingFence', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'heal-binding-fence'; + const sub = { + subscribed: true, + onChainId: '321', + lastReconciledOrdinal: 0, + }; + internals.subscribedContextGraphs.set(localCgId, sub); + let releaseHeal!: () => void; + let markHealStarted!: () => void; + const healStarted = new Promise((resolve) => { markHealStarted = resolve; }); + const healRelease = new Promise((resolve) => { releaseHeal = resolve; }); + (internals as any).healStrandedScopedKCs = async () => { + markHealStarted(); + await healRelease; + }; + const getCount = vi.fn(async () => 0n); + chain.getContextGraphKCCount = getCount; + + const reconcile = (internals as any).executeVmReconcileForCg(localCgId, 'manual'); + await healStarted; + sub.onChainId = '322'; + releaseHeal(); + + await expect(reconcile).rejects.toMatchObject({ name: 'VmReconcileQueueClosedError' }); + expect(getCount).not.toHaveBeenCalled(); + expect(sub.lastReconciledOrdinal).toBe(0); + }); + + it('retires an aborted reconcile even when stranded-KC repair ignores cancellation', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillHealAbortRace', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'heal-abort-race'; + internals.subscribedContextGraphs.set(localCgId, { + subscribed: true, + onChainId: '323', + lastReconciledOrdinal: 0, + }); + let markHealStarted!: () => void; + const healStarted = new Promise((resolve) => { markHealStarted = resolve; }); + let healCalls = 0; + (internals as any).healStrandedScopedKCs = async () => { + healCalls += 1; + if (healCalls === 1) { + markHealStarted(); + await new Promise(() => undefined); + } + }; + chain.getContextGraphKCCount = async () => 0n; + + const abandoned = (internals as any).executeVmReconcileForCg(localCgId, 'manual'); + await healStarted; + (internals as any).closeVmReconcileRotationState(); + await expect(abandoned).rejects.toMatchObject({ name: 'VmReconcileQueueClosedError' }); + + (internals as any).openVmReconcileRotationState(); + await expect((internals as any).executeVmReconcileForCg(localCgId, 'manual')) + .resolves.toMatchObject({ status: 'current' }); + expect(healCalls).toBe(2); + }); + + it('keeps watermark persistence inside the tracked physical reconcile', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillWatermarkDrain', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'watermark-drain'; + internals.subscribedContextGraphs.set(localCgId, { + subscribed: true, + onChainId: '324', + lastReconciledOrdinal: 0, + }); + chain.getContextGraphKCCount = async () => 1n; + (internals as any).healStrandedScopedKCs = async () => undefined; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + let markPersistStarted!: () => void; + const persistStarted = new Promise((resolve) => { markPersistStarted = resolve; }); + let releasePersist!: () => void; + const persistGate = new Promise((resolve) => { releasePersist = resolve; }); + (internals as any).persistVmReconcileWatermark = async () => { + markPersistStarted(); + await persistGate; + }; + + const reconcile = (internals as any).executeVmReconcileForCg(localCgId, 'manual'); + await persistStarted; + expect((internals as any).vmReconcilePhysicalRuns.size).toBe(1); + let settled = false; + void reconcile.finally(() => { settled = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(settled).toBe(false); + + releasePersist(); + await expect(reconcile).resolves.toMatchObject({ watermarkAfter: 1 }); + expect((internals as any).vmReconcilePhysicalRuns.size).toBe(0); + }); + + it('does not expose an advanced watermark when its durable save fails', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillWatermarkSaveFailure', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'watermark-save-failure'; + const subscription = { + subscribed: true, + onChainId: '325', + lastReconciledOrdinal: 0, + }; + internals.subscribedContextGraphs.set(localCgId, subscription); + chain.getContextGraphKCCount = async () => 1n; + (internals as any).healStrandedScopedKCs = async () => undefined; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + (internals as any).persistVmReconcileWatermark = async () => { + throw new Error('subscription store unavailable'); + }; + + await expect((internals as any).executeVmReconcileForCg(localCgId, 'manual')) + .rejects.toThrow('subscription store unavailable'); + expect(subscription.lastReconciledOrdinal).toBe(0); + expect((internals as any).reconcileCursors.get(localCgId)?.watermark).toBe(0); + }); + + it('flushes reconcile materialization before strictly saving its watermark', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillWatermarkFlushOrder', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'watermark-flush-order'; + const subscription = { + subscribed: true, + onChainId: '326', + lastReconciledOrdinal: 0, + }; + internals.subscribedContextGraphs.set(localCgId, subscription); + chain.getContextGraphKCCount = async () => 1n; + (internals as any).healStrandedScopedKCs = async () => undefined; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + const durabilityOrder: string[] = []; + internals.store.flush = async () => { durabilityOrder.push('flush'); }; + (internals as any).persistContextGraphSubscriptionStrict = async () => { + durabilityOrder.push('save'); + }; + + await expect((internals as any).executeVmReconcileForCg(localCgId, 'manual')) + .resolves.toMatchObject({ watermarkAfter: 1 }); + expect(durabilityOrder).toEqual(['flush', 'save']); + expect(subscription.lastReconciledOrdinal).toBe(1); + }); + + it('flushes reconcile materialization while confirmation depth holds the watermark', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillHeldWatermarkFlush', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'held-watermark-flush'; + const subscription = { + subscribed: true, + onChainId: '330', + lastReconciledOrdinal: 0, + }; + internals.subscribedContextGraphs.set(localCgId, subscription); + chain.getContextGraphKCCount = async () => 1n; + chain.getBlockNumber = async () => 100; + (internals as any).healStrandedScopedKCs = async () => undefined; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + const flush = vi.fn(async () => undefined); + internals.store.flush = flush; + const persistStrict = vi.fn(async () => undefined); + (internals as any).persistContextGraphSubscriptionStrict = persistStrict; + + await expect((internals as any).executeVmReconcileForCg(localCgId, 'manual')) + .resolves.toMatchObject({ watermarkAfter: 0, reconciledOrdinals: 1 }); + expect(flush).toHaveBeenCalledOnce(); + expect(persistStrict).not.toHaveBeenCalled(); + expect(subscription.lastReconciledOrdinal).toBe(0); + }); + + it('does not save or expose a watermark when reconcile materialization flush fails', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillWatermarkFlushFailure', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'watermark-flush-failure'; + const subscription = { + subscribed: true, + onChainId: '327', + lastReconciledOrdinal: 0, + }; + internals.subscribedContextGraphs.set(localCgId, subscription); + chain.getContextGraphKCCount = async () => 1n; + (internals as any).healStrandedScopedKCs = async () => undefined; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + internals.store.flush = async () => { throw new Error('triple-store flush failed'); }; + const persistStrict = vi.fn(async () => undefined); + (internals as any).persistContextGraphSubscriptionStrict = persistStrict; + + await expect((internals as any).executeVmReconcileForCg(localCgId, 'manual')) + .rejects.toThrow('triple-store flush failed'); + expect(persistStrict).not.toHaveBeenCalled(); + expect(subscription.lastReconciledOrdinal).toBe(0); + expect((internals as any).reconcileCursors.get(localCgId)?.watermark).toBe(0); + }); + + it('fences a watermark save continuation across a same-object binding ABA', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'CoreFillWatermarkBindingAba', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'watermark-binding-aba'; + const subscription = { + subscribed: true, + onChainId: '328', + lastReconciledOrdinal: 0, + }; + internals.subscribedContextGraphs.set(localCgId, subscription); + chain.getContextGraphKCCount = async () => 1n; + (internals as any).healStrandedScopedKCs = async () => undefined; + (internals as any).reconcileChainOrdinal = async () => ({ + status: 'reconciled', blockNumber: 100, + }); + internals.store.flush = async () => undefined; + let markSaveStarted!: () => void; + const saveStarted = new Promise((resolve) => { markSaveStarted = resolve; }); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + (internals as any).persistContextGraphSubscriptionStrict = async () => { + markSaveStarted(); + await saveGate; + }; + + const reconcile = (internals as any).executeVmReconcileForCg(localCgId, 'manual'); + await saveStarted; + const originalCursor = (internals as any).reconcileCursors.get(localCgId); + (internals as any).bindSubscriptionOnChainId(localCgId, subscription, '329'); + (internals as any).bindSubscriptionOnChainId(localCgId, subscription, '328'); + (internals as any).reconcileCursors.set(localCgId, originalCursor); + releaseSave(); + + await expect(reconcile).rejects.toMatchObject({ name: 'VmReconcileQueueClosedError' }); + expect(subscription.onChainId).toBe('328'); + expect(subscription.lastReconciledOrdinal).toBe(0); + expect(originalCursor.watermark).toBe(0); + }); + it('reports a durable watermark ahead of the chain head without ordinal work', async () => { const chain = new MockChainAdapter(); agent = await DKGAgent.create({ name: 'CoreFillWatermarkAhead', chainAdapter: chain }); diff --git a/packages/agent/test/discovery-peer-pagination.test.ts b/packages/agent/test/discovery-peer-pagination.test.ts new file mode 100644 index 0000000000..6f6fcb577e --- /dev/null +++ b/packages/agent/test/discovery-peer-pagination.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from 'vitest'; +import { DKGQueryEngine } from '@origintrail-official/dkg-query'; +import { OxigraphStore } from '@origintrail-official/dkg-storage'; +import { DiscoveryClient } from '../src/discovery.js'; +import { buildAgentProfile } from '../src/profile.js'; + +describe('DiscoveryClient curator peer pagination', () => { + it('queries distinct peer IDs in deterministic exclusive-cursor order', async () => { + const query = vi.fn(async () => ({ + type: 'bindings' as const, + bindings: [ + { peerId: '"peer-011"' }, + { peerId: '"peer-012"' }, + ], + })); + const discovery = new DiscoveryClient({ query } as any); + + await expect(discovery.findAgentPeerIdsByAddress( + '0xabc', + { afterPeerId: 'peer-010', limit: 2 }, + )).resolves.toEqual(['peer-011', 'peer-012']); + + const [sparql, options] = query.mock.calls[0]!; + expect(sparql).toContain('SELECT DISTINCT ?peerId'); + expect(sparql).toContain('FILTER(STR(?peerId) > "peer-010")'); + expect(sparql).toContain('ORDER BY ASC(STR(?peerId))'); + expect(sparql).toContain('LIMIT 2'); + expect(options).toMatchObject({ contextGraphId: 'agents' }); + }); + + it('pages real agent-registry data without duplicate profile rows', async () => { + const store = new OxigraphStore(); + const curator = '0x00000000000000000000000000000000000000ab'; + try { + const first = buildAgentProfile({ + peerId: 'peer-001', name: 'First', agentAddress: curator, skills: [], + }); + const second = buildAgentProfile({ + peerId: 'peer-002', name: 'Second', agentAddress: curator, skills: [], + }); + await store.insert([...first.quads, ...second.quads]); + const discovery = new DiscoveryClient(new DKGQueryEngine(store)); + + await expect(discovery.findAgentPeerIdsByAddress(curator, { limit: 1 })) + .resolves.toEqual(['peer-001']); + await expect(discovery.findAgentPeerIdsByAddress( + curator, + { afterPeerId: 'peer-001', limit: 2 }, + )).resolves.toEqual(['peer-002']); + } finally { + await store.close(); + } + }); + + it('matches legacy mixed-case EVM wallet rows case-insensitively', async () => { + const store = new OxigraphStore(); + const lowerAddress = '0xabcdef00000000000000000000000000000000ab'; + const mixedAddress = '0xAbCdEf00000000000000000000000000000000aB'; + try { + const profile = buildAgentProfile({ + peerId: 'peer-checksum', name: 'Checksum Curator', agentAddress: lowerAddress, skills: [], + }); + const legacyQuads = profile.quads.map((quad) => ( + quad.predicate === 'https://dkg.network/ontology#agentAddress' + ? { ...quad, object: `"${mixedAddress}"` } + : quad + )); + await store.insert(legacyQuads); + const discovery = new DiscoveryClient(new DKGQueryEngine(store)); + + await expect(discovery.findAgentPeerIdsByAddress(lowerAddress)) + .resolves.toEqual(['peer-checksum']); + } finally { + await store.close(); + } + }); +}); diff --git a/packages/agent/test/durable-sync-graph-scoped-materialization.test.ts b/packages/agent/test/durable-sync-graph-scoped-materialization.test.ts index a83f91c6b5..fa8f968e99 100644 --- a/packages/agent/test/durable-sync-graph-scoped-materialization.test.ts +++ b/packages/agent/test/durable-sync-graph-scoped-materialization.test.ts @@ -1573,6 +1573,112 @@ describe('durable graph-scoped KA materialization', () => { expect(await values(store, 'assertionVersion')).toEqual(['"1"']); }); + it('does not pass a lifecycle abort signal into an entered atomic replacement', async () => { + const store = new OxigraphStore(); + const controller = new AbortController(); + let releaseReplace!: () => void; + let markReplaceEntered!: () => void; + const replaceEntered = new Promise((resolve) => { markReplaceEntered = resolve; }); + const replaceGate = new Promise((resolve) => { releaseReplace = resolve; }); + const replaceGraphAndSubject = store.replaceGraphAndSubject!.bind(store); + let observedCommitSignal: AbortSignal | undefined; + const gatedStore = new Proxy(store, { + get(target, property) { + if (property === 'replaceGraphAndSubject') { + return async (...args: Parameters>) => { + observedCommitSignal = args[5]?.signal; + markReplaceEntered(); + await replaceGate; + if (observedCommitSignal?.aborted) { + throw Object.assign(new Error('transport aborted'), { name: 'AbortError' }); + } + return replaceGraphAndSubject(...args); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }) as TripleStore; + let current = true; + let settled = false; + const materialization = materializeVerifiedGraphScopedAsset({ + store: gatedStore, + asset: { + contextGraphId, + ual, + assertionVersion: 2n, + assertionGraph, + metaGraph, + dataQuads: [dataQuad(2)], + metadataQuads: metadata(2), + }, + isCurrent: () => current, + options: { signal: controller.signal }, + }).finally(() => { settled = true; }); + + await replaceEntered; + current = false; + controller.abort(); + await Promise.resolve(); + expect(settled).toBe(false); + expect(observedCommitSignal).toBeUndefined(); + + releaseReplace(); + await expect(materialization).resolves.toBe('quarantined'); + expect(await values(store, 'assertionVersion')).toEqual(['"2"']); + }); + + it('atomically removes an asset when its subscription is deleted after commit starts', async () => { + const store = new OxigraphStore(); + let releaseReplace!: () => void; + let markReplaceCommitted!: () => void; + const replaceCommitted = new Promise((resolve) => { markReplaceCommitted = resolve; }); + const replaceGate = new Promise((resolve) => { releaseReplace = resolve; }); + const replaceGraphAndSubject = store.replaceGraphAndSubject!.bind(store); + let replaceCalls = 0; + const gatedStore = new Proxy(store, { + get(target, property) { + if (property === 'replaceGraphAndSubject') { + return async (...args: Parameters>) => { + replaceCalls += 1; + const result = await replaceGraphAndSubject(...args); + if (replaceCalls === 1) { + markReplaceCommitted(); + await replaceGate; + } + return result; + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }) as TripleStore; + let subscriptionPresent = true; + const materialization = materializeVerifiedGraphScopedAsset({ + store: gatedStore, + asset: { + contextGraphId, + ual, + assertionVersion: 2n, + assertionGraph, + metaGraph, + dataQuads: [dataQuad(2)], + metadataQuads: metadata(2), + }, + isCurrent: () => true, + shouldQuarantineCommitted: () => !subscriptionPresent, + }); + + await replaceCommitted; + subscriptionPresent = false; + releaseReplace(); + + await expect(materialization).resolves.toBe('quarantined'); + expect(replaceCalls).toBe(2); + expect(await store.countQuads(assertionGraph)).toBe(0); + expect(await values(store, 'assertionVersion')).toEqual([]); + }); + it('leaves both old partitions intact when the atomic store update fails', async () => { const store = new OxigraphStore(); const v1Data = dataQuad(1); diff --git a/packages/agent/test/durable-sync-lifecycle-binding.test.ts b/packages/agent/test/durable-sync-lifecycle-binding.test.ts index 2a4eec0c6a..2a59d79953 100644 --- a/packages/agent/test/durable-sync-lifecycle-binding.test.ts +++ b/packages/agent/test/durable-sync-lifecycle-binding.test.ts @@ -5,7 +5,11 @@ import type { OperationContext } from '@origintrail-official/dkg-core'; vi.mock('../src/sync/requester/durable-sync.js', async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, runDurableSync: vi.fn(async () => ({})) }; + return { + ...actual, + runDurableSync: vi.fn(async () => ({})), + runDurableSyncDetailed: vi.fn(async () => ({ result: {} })), + }; }); vi.mock('../src/sync/requester/graph-scoped-materialization.js', async (importOriginal) => { @@ -24,6 +28,7 @@ import { DKGAgent } from '../src/dkg-agent.js'; import { LifecycleSyncMethods } from '../src/dkg-agent-lifecycle.js'; import { runDurableSync, + runDurableSyncDetailed, type DurableSyncContext, type DurableSyncGraphScopedStoreRequest, } from '../src/sync/requester/durable-sync.js'; @@ -40,6 +45,7 @@ const metaGraph = `did:dkg:context-graph:${contextGraphId}/_meta`; const ctx = { kind: 'sync', id: 'lifecycle-binding-test', startedAt: 0 } as OperationContext; const mockedRunDurableSync = vi.mocked(runDurableSync); +const mockedRunDurableSyncDetailed = vi.mocked(runDurableSyncDetailed); const mockedMaterialize = vi.mocked(materializeVerifiedGraphScopedAsset); function graphScopedAsset( @@ -90,6 +96,7 @@ async function captureGraphScopedStore( totalTimeoutMs?: number; signal?: AbortSignal; onAtomicCommitStarted?: (contextGraphId: string, ual: string) => void; + onAgentLike?: (agentLike: any) => void; } = {}, ) { const agentLike: any = { @@ -97,9 +104,12 @@ async function captureGraphScopedStore( chain, store: {}, subscribedContextGraphs: new Map(), + contextGraphBindingGenerations: new Map(), wireIdToLocalCgId: new Map(), + graphScopedStoreClosed: false, + graphScopedStorePhysicalRuns: new Set>(), bindSubscriptionOnChainId: vi.fn(), - persistContextGraphSubscriptionState: vi.fn(), + persistContextGraphSubscriptionStrict: vi.fn(), processDurableBatchInWorker: async () => ({}), insertSyncedQuadsAndInvalidateListCache: async () => {}, syncCheckpoints: new Map(), @@ -114,6 +124,7 @@ async function captureGraphScopedStore( ).requireLocalCgMatchesOnChainSlot; agentLike.isWireIdKeyedSubscription = (DKGAgent.prototype as any).isWireIdKeyedSubscription; agentLike.raceChainPolicyRead = (DKGAgent.prototype as any).raceChainPolicyRead; + options.onAgentLike?.(agentLike); await LifecycleSyncMethods.prototype.runLegacyDurableSyncForContextGraph.call( agentLike, @@ -138,6 +149,7 @@ async function captureGraphScopedStore( describe('durable sync lifecycle chain binding', () => { beforeEach(() => { mockedRunDurableSync.mockClear(); + mockedRunDurableSyncDetailed.mockClear(); mockedMaterialize.mockClear(); }); @@ -288,9 +300,10 @@ describe('durable sync lifecycle chain binding', () => { chain: { chainId: 'none' }, store: {}, subscribedContextGraphs: new Map(), + contextGraphBindingGenerations: new Map(), wireIdToLocalCgId: new Map(), bindSubscriptionOnChainId: vi.fn(), - persistContextGraphSubscriptionState: vi.fn(), + persistContextGraphSubscriptionStrict: vi.fn(), processDurableBatchInWorker: async () => ({}), insertSyncedQuadsAndInvalidateListCache, syncCheckpoints: new Map(), @@ -321,19 +334,25 @@ describe('durable sync lifecycle chain binding', () => { }); it('selects the dedicated field-sized exact-recovery transfer policy', async () => { - const runLegacyDurableSync = vi.fn(async () => ({})); - const agentLike = { runLegacyDurableSync }; + const physicalResult = {} as Awaited>; + const runLegacyDurableSyncDetailed = vi.fn(async () => ({ + result: physicalResult, + exactFetchDisposition: 'clean-absent' as const, + })); + const agentLike = { runLegacyDurableSyncDetailed }; const exactUal = 'did:dkg:base:84532/0x1111111111111111111111111111111111111111/1'; + const controller = new AbortController(); - await LifecycleSyncMethods.prototype.syncExactKnowledgeAssetsFromPeer.call( + const detailed = await LifecycleSyncMethods.prototype.syncExactKnowledgeAssetsFromPeerDetailed.call( agentLike as any, '12D3KooWExactRecoveryPeer', '0x1111111111111111111111111111111111111111/blackbox', [exactUal], + { signal: controller.signal }, ); - expect(runLegacyDurableSync).toHaveBeenCalledTimes(1); - expect(runLegacyDurableSync.mock.calls[0]?.[6]).toMatchObject({ + expect(runLegacyDurableSyncDetailed).toHaveBeenCalledTimes(1); + expect(runLegacyDurableSyncDetailed.mock.calls[0]?.[6]).toMatchObject({ exactAssetUals: [exactUal], stopOnBackoffWorthyFailure: true, priority: 1_000, @@ -342,8 +361,34 @@ describe('durable sync lifecycle chain binding', () => { // the whole point of the label. Without this line, deleting it from the // call site keeps every test green and only the Grafana attribution rots. source: 'vm-recovery', + signal: controller.signal, }); - expect(runLegacyDurableSync.mock.calls[0]?.[6]).not.toHaveProperty('totalTimeoutMs'); + expect(runLegacyDurableSyncDetailed.mock.calls[0]?.[6]).not.toHaveProperty('totalTimeoutMs'); + expect(detailed).toEqual({ result: physicalResult, disposition: 'clean-absent' }); + }); + + it('projects the public exact-sync result from the detailed implementation', async () => { + const result = {} as Awaited>; + const syncExactKnowledgeAssetsFromPeerDetailed = vi.fn(async () => ({ + result, + disposition: 'found' as const, + })); + const requestedAssetUals = [ual]; + + const projected = await LifecycleSyncMethods.prototype.syncExactKnowledgeAssetsFromPeer.call( + { syncExactKnowledgeAssetsFromPeerDetailed } as any, + '12D3KooWExactProjectionPeer', + contextGraphId, + requestedAssetUals, + ); + + expect(syncExactKnowledgeAssetsFromPeerDetailed).toHaveBeenCalledWith( + '12D3KooWExactProjectionPeer', + contextGraphId, + requestedAssetUals, + {}, + ); + expect(projected).toBe(result); }); it.each([ @@ -480,29 +525,70 @@ describe('durable sync lifecycle chain binding', () => { totalTimeoutMs: 30_000, }, ); - expect(mockedRunDurableSync).toHaveBeenCalledTimes(1); + expect(mockedRunDurableSyncDetailed).toHaveBeenCalledTimes(1); expect( - mockedRunDurableSync.mock.calls[0]![0].durableSyncBudget + mockedRunDurableSyncDetailed.mock.calls[0]![0].durableSyncBudget .createContextGraphBudget({ contextGraphId, remainingContextGraphs: 1 }) .fetchDeadline, ).toBe(1_800_000_030_000); - mockedRunDurableSync.mockClear(); - agentLike.runLegacyDurableSync = LifecycleSyncMethods.prototype.runLegacyDurableSync; + mockedRunDurableSyncDetailed.mockClear(); + agentLike.runLegacyDurableSyncDetailed = LifecycleSyncMethods.prototype.runLegacyDurableSyncDetailed; + agentLike.syncExactKnowledgeAssetsFromPeerDetailed = + LifecycleSyncMethods.prototype.syncExactKnowledgeAssetsFromPeerDetailed; await LifecycleSyncMethods.prototype.syncExactKnowledgeAssetsFromPeer.call( agentLike, 'peer-internal-exact-recovery', contextGraphId, [exactUal], ); - expect(mockedRunDurableSync).toHaveBeenCalledTimes(1); + expect(mockedRunDurableSyncDetailed).toHaveBeenCalledTimes(1); expect( - mockedRunDurableSync.mock.calls[0]![0].durableSyncBudget + mockedRunDurableSyncDetailed.mock.calls[0]![0].durableSyncBudget .createContextGraphBudget({ contextGraphId, remainingContextGraphs: 1 }) .fetchDeadline, ).toBe(1_800_000_600_000); }); + it('keeps a multi-graph exact lifecycle result incomplete after a later clean absence', async () => { + const exactUal = 'did:dkg:base:84532/0x1111111111111111111111111111111111111111/1'; + const agentLike: any = { + config: {}, + processDurableBatchInWorker: async () => ({}), + runContextGraphSyncWithBackpressure: async ( + _ctx: unknown, + _contextGraphId: string, + _lane: string, + _operationId: string, + work: () => Promise, + ) => work(), + log: { info: () => {}, warn: () => {}, debug: () => {} }, + }; + mockedRunDurableSyncDetailed + .mockResolvedValueOnce({ + result: {} as Awaited>, + exactFetchDisposition: 'incomplete', + }) + .mockResolvedValueOnce({ + result: {} as Awaited>, + exactFetchDisposition: 'clean-absent', + }); + + const detailed = await LifecycleSyncMethods.prototype.runLegacyDurableSyncDetailed.call( + agentLike, + ctx, + 'peer-multi-exact', + ['exact-incomplete-cg', 'exact-clean-cg'], + undefined, + undefined, + undefined, + { exactAssetUals: [exactUal] }, + ); + + expect(mockedRunDurableSyncDetailed).toHaveBeenCalledTimes(2); + expect(detailed.exactFetchDisposition).toBe('incomplete'); + }); + it('keeps caller-signalled durable sync off the non-cancellable changelog lane', async () => { const runChangelogLane = vi.fn(async () => ({ remainingLegacyCgs: [] })); const runLegacyDurableSync = vi.fn(async () => ({ @@ -581,16 +667,19 @@ describe('durable sync lifecycle chain binding', () => { sub.onChainId = onChainId; }, ); - const persistContextGraphSubscriptionState = vi.fn(); + const persistContextGraphSubscriptionStrict = vi.fn(); const onAtomicCommitStarted = vi.fn(); const agentLike: any = { config: {}, chain, store: {}, subscribedContextGraphs: new Map([[contextGraphId, subscription]]), + contextGraphBindingGenerations: new Map(), wireIdToLocalCgId: new Map(), + graphScopedStoreClosed: false, + graphScopedStorePhysicalRuns: new Set>(), bindSubscriptionOnChainId, - persistContextGraphSubscriptionState, + persistContextGraphSubscriptionStrict, processDurableBatchInWorker: async () => ({}), insertSyncedQuadsAndInvalidateListCache: async () => {}, syncCheckpoints: new Map(), @@ -649,7 +738,7 @@ describe('durable sync lifecycle chain binding', () => { await vi.advanceTimersByTimeAsync(0); expect(getContextGraphNameHash).toHaveBeenCalledTimes(1); expect(bindSubscriptionOnChainId).not.toHaveBeenCalled(); - expect(persistContextGraphSubscriptionState).not.toHaveBeenCalled(); + expect(persistContextGraphSubscriptionStrict).not.toHaveBeenCalled(); expect(onAtomicCommitStarted).not.toHaveBeenCalled(); expect(mockedMaterialize).not.toHaveBeenCalled(); expect(agentLike.invalidateListContextGraphsCache).not.toHaveBeenCalled(); @@ -675,11 +764,16 @@ describe('durable sync lifecycle chain binding', () => { '14', ); expect(subscription.onChainId).toBe('14'); - expect(persistContextGraphSubscriptionState).toHaveBeenCalledWith(contextGraphId); + expect(persistContextGraphSubscriptionStrict).toHaveBeenCalledWith( + contextGraphId, + expect.objectContaining({ onChainId: '14', lastReconciledOrdinal: 0 }), + undefined, + expect.any(Function), + ); expect(bindSubscriptionOnChainId.mock.invocationCallOrder[0]).toBeLessThan( mockedMaterialize.mock.invocationCallOrder[0]!, ); - expect(persistContextGraphSubscriptionState.mock.invocationCallOrder[0]).toBeLessThan( + expect(persistContextGraphSubscriptionStrict.mock.invocationCallOrder[0]).toBeLessThan( mockedMaterialize.mock.invocationCallOrder[0]!, ); expect(onAtomicCommitStarted.mock.invocationCallOrder[0]).toBeLessThan( @@ -691,18 +785,112 @@ describe('durable sync lifecycle chain binding', () => { unknown >; expect('verifiedOnChainContextGraphId' in materializedAsset).toBe(false); + const shouldQuarantineCommitted = mockedMaterialize.mock.calls[0]![0] + .shouldQuarantineCommitted; + expect(shouldQuarantineCommitted?.()).toBe(false); + subscription.onChainId = undefined; + expect(shouldQuarantineCommitted?.()).toBe(true); + subscription.onChainId = '14'; + agentLike.subscribedContextGraphs.delete(contextGraphId); + expect(shouldQuarantineCommitted?.()).toBe(true); + agentLike.subscribedContextGraphs.set(contextGraphId, subscription); + + subscription.onChainId = undefined; + persistContextGraphSubscriptionStrict.mockRejectedValueOnce(new Error('subscription store unavailable')); + const bindsBeforeRejectedSave = bindSubscriptionOnChainId.mock.calls.length; + const materializationsBeforeRejectedSave = mockedMaterialize.mock.calls.length; + await expect(storeGraphScopedAsset!( + graphScopedStoreRequest(asset, Date.now() + 60_000), + )).rejects.toThrow('subscription store unavailable'); + expect(subscription.onChainId).toBeUndefined(); + expect(bindSubscriptionOnChainId).toHaveBeenCalledTimes(bindsBeforeRejectedSave); + expect(mockedMaterialize).toHaveBeenCalledTimes(materializationsBeforeRejectedSave); }); - it('does not reuse a binding proof across different on-chain CG slots', async () => { + it('retains a chain-authenticated public asset when no subscription exists', async () => { + const root = new Uint8Array(32); + root[31] = 2; + const chain = { + chainId: 'otp:2043', + getLatestMerkleRoot: async () => root, + getMerkleRootCount: async () => 2n, + getKAContextGraphId: async () => 14n, + getContextGraphNameHash: async () => ( + ethers.keccak256(ethers.toUtf8Bytes(contextGraphId)) + ), + getLatestMerkleRootPublisher: async () => '0x2222222222222222222222222222222222222222', + verifyKAUpdate: async () => ({ + verified: true, + onChainMerkleRoot: root, + blockNumber: 123, + txIndex: 4, + merkleRootCount: 2n, + }), + } as ChainAdapter; + + const storeGraphScopedAsset = await captureGraphScopedStore(chain); + await expect(storeGraphScopedAsset( + graphScopedStoreRequest(graphScopedAsset(root), Date.now() + 60_000), + )).resolves.toBe('applied'); + + expect(mockedMaterialize).toHaveBeenCalledOnce(); + expect(mockedMaterialize.mock.calls[0]![0].shouldQuarantineCommitted?.()).toBe(false); + }); + + it('does not let a stale strict snapshot overwrite a newer host-only persistence write', async () => { + const oldSubscription = { subscribed: true, onChainId: '14' }; + const hostOnlySubscription = { subscribed: false, coreHosted: true, onChainId: '14' }; + let durableRecord: Record | undefined; + let releaseHostWrite!: () => void; + const hostWriteGate = new Promise((resolve) => { releaseHostWrite = resolve; }); + let persistChain = Promise.resolve(); + const enqueueContextGraphSubscriptionPersistWrite = ( + _contextGraphId: string, + write: () => Promise, + ) => { + const run = persistChain.then(write); + persistChain = run.catch(() => undefined); + return run; + }; + const agentLike: any = { + config: { + contextGraphSubscriptionStore: { + loadAll: async () => [], + save: async (record: Record) => { durableRecord = { ...record }; }, + delete: async () => { durableRecord = undefined; }, + }, + }, + subscribedContextGraphs: new Map([[contextGraphId, oldSubscription]]), + contextGraphBindingGenerations: new Map(), + enqueueContextGraphSubscriptionPersistWrite, + }; + + const capturedSubscription = oldSubscription; + agentLike.subscribedContextGraphs.set(contextGraphId, hostOnlySubscription); + const hostWrite = enqueueContextGraphSubscriptionPersistWrite(contextGraphId, async () => { + await hostWriteGate; + durableRecord = { id: contextGraphId, ...hostOnlySubscription }; + }); + const staleStrictWrite = LifecycleSyncMethods.prototype.persistContextGraphSubscriptionStrict.call( + agentLike, + contextGraphId, + { ...capturedSubscription, onChainId: '15', lastReconciledOrdinal: 0 }, + undefined, + () => agentLike.subscribedContextGraphs.get(contextGraphId) === capturedSubscription, + ); + + releaseHostWrite(); + await hostWrite; + await expect(staleStrictWrite).rejects.toThrow(/changed before.*persisted/); + expect(durableRecord).toEqual({ id: contextGraphId, ...hostOnlySubscription }); + }); + + it('does not reuse a proof or roll an authoritative binding across same-name CG slots', async () => { const root = new Uint8Array(32); root[31] = 2; const rootHex = Array.from(root, (byte) => byte.toString(16).padStart(2, '0')).join(''); const expectedNameHash = ethers.keccak256(ethers.toUtf8Bytes(contextGraphId)); - const getContextGraphNameHash = vi.fn(async (onChainId: bigint) => ( - onChainId === 14n - ? expectedNameHash - : ethers.keccak256(ethers.toUtf8Bytes('different-context-graph')) - )); + const getContextGraphNameHash = vi.fn(async () => expectedNameHash); const kaNumberMask = (1n << 96n) - 1n; const chain = { chainId: 'otp:2043', @@ -727,13 +915,16 @@ describe('durable sync lifecycle chain binding', () => { chain, store: {}, subscribedContextGraphs: new Map([[contextGraphId, subscription]]), + contextGraphBindingGenerations: new Map(), wireIdToLocalCgId: new Map(), + graphScopedStoreClosed: false, + graphScopedStorePhysicalRuns: new Set>(), bindSubscriptionOnChainId: vi.fn( (_localId: string, sub: typeof subscription, onChainId: string) => { sub.onChainId = onChainId; }, ), - persistContextGraphSubscriptionState: vi.fn(), + persistContextGraphSubscriptionStrict: vi.fn(), processDurableBatchInWorker: async () => ({}), insertSyncedQuadsAndInvalidateListCache: async () => {}, syncCheckpoints: new Map(), @@ -794,6 +985,9 @@ describe('durable sync lifecycle chain binding', () => { expect(getContextGraphNameHash).toHaveBeenCalledTimes(2); expect(getContextGraphNameHash.mock.calls.map(([id]) => id)).toEqual([14n, 15n]); + expect(agentLike.persistContextGraphSubscriptionStrict).toHaveBeenCalledOnce(); + expect(agentLike.bindSubscriptionOnChainId).toHaveBeenCalledOnce(); + expect(subscription.onChainId).toBe('14'); expect(mockedMaterialize).toHaveBeenCalledTimes(1); }); @@ -851,6 +1045,123 @@ describe('durable sync lifecycle chain binding', () => { expect(mockedMaterialize).not.toHaveBeenCalled(); }); + it('registers the physical store before invoking a pluggable chain adapter', async () => { + const root = new Uint8Array(32); + let agentLike: any; + let physicalRunsSeenByAdapter = -1; + const chain = { + chainId: 'otp:2043', + getLatestMerkleRoot: async () => { + physicalRunsSeenByAdapter = agentLike.graphScopedStorePhysicalRuns.size; + return root; + }, + getMerkleRootCount: async () => 2n, + getKAContextGraphId: async () => 14n, + getContextGraphNameHash: async () => ethers.keccak256(ethers.toUtf8Bytes(contextGraphId)), + getLatestMerkleRootPublisher: async () => '0x2222222222222222222222222222222222222222', + verifyKAUpdate: async () => ({ + verified: true, + onChainMerkleRoot: root, + blockNumber: 123, + txIndex: 4, + merkleRootCount: 2n, + }), + } as ChainAdapter; + const storeGraphScopedAsset = await captureGraphScopedStore(chain, vi.fn(), { + onAgentLike: (captured) => { agentLike = captured; }, + }); + + const pending = storeGraphScopedAsset(graphScopedStoreRequest( + graphScopedAsset(root), + Date.now() + 120_000, + )); + expect(agentLike.graphScopedStorePhysicalRuns.size).toBe(1); + await expect(pending).resolves.toBe('applied'); + expect(physicalRunsSeenByAdapter).toBe(1); + expect(agentLike.graphScopedStorePhysicalRuns.size).toBe(0); + }); + + it('rejects an ordinary durable asset when its subscription rebinds during authentication', async () => { + const root = new Uint8Array(32); + let releaseRoot!: () => void; + let markRootReadStarted!: () => void; + const rootReadStarted = new Promise((resolve) => { markRootReadStarted = resolve; }); + const rootGate = new Promise((resolve) => { releaseRoot = resolve; }); + const chain = { + chainId: 'otp:2043', + getLatestMerkleRoot: async () => { + markRootReadStarted(); + await rootGate; + return root; + }, + getMerkleRootCount: async () => 2n, + getKAContextGraphId: async () => 14n, + getContextGraphNameHash: async () => ethers.keccak256(ethers.toUtf8Bytes(contextGraphId)), + getLatestMerkleRootPublisher: async () => '0x2222222222222222222222222222222222222222', + verifyKAUpdate: async () => ({ + verified: true, + onChainMerkleRoot: root, + blockNumber: 123, + txIndex: 4, + merkleRootCount: 2n, + }), + } as ChainAdapter; + const subscription = { subscribed: true, onChainId: '14', lastReconciledOrdinal: 9 }; + const bindSubscriptionOnChainId = vi.fn(); + const persistContextGraphSubscriptionStrict = vi.fn(); + const syncCheckpoints = new Map([['unchanged', 17]]); + const agentLike: any = { + config: {}, + chain, + store: {}, + subscribedContextGraphs: new Map([[contextGraphId, subscription]]), + contextGraphBindingGenerations: new Map(), + wireIdToLocalCgId: new Map(), + graphScopedStoreClosed: false, + graphScopedStorePhysicalRuns: new Set>(), + bindSubscriptionOnChainId, + persistContextGraphSubscriptionStrict, + processDurableBatchInWorker: async () => ({}), + insertSyncedQuadsAndInvalidateListCache: async () => {}, + syncCheckpoints, + oversizeTombstoneLog: { record: () => {} }, + invalidateListContextGraphsCache: vi.fn(), + contextGraphMetaProjection: { markDirtyFromQuads: vi.fn() }, + log: { info: () => {}, warn: () => {}, debug: () => {} }, + }; + agentLike.localCgMatchesOnChainSlot = (DKGAgent.prototype as any).localCgMatchesOnChainSlot; + agentLike.requireLocalCgMatchesOnChainSlot = ( + DKGAgent.prototype as any + ).requireLocalCgMatchesOnChainSlot; + agentLike.isWireIdKeyedSubscription = (DKGAgent.prototype as any).isWireIdKeyedSubscription; + agentLike.raceChainPolicyRead = (DKGAgent.prototype as any).raceChainPolicyRead; + await LifecycleSyncMethods.prototype.runLegacyDurableSyncForContextGraph.call( + agentLike, + ctx, + 'peer-stale-exact', + contextGraphId, + 1, + ); + const storeGraphScopedAsset = mockedRunDurableSync.mock.calls[0]![0] + .storeGraphScopedAsset!; + const pending = storeGraphScopedAsset(graphScopedStoreRequest( + graphScopedAsset(root), + Date.now() + 120_000, + )); + await rootReadStarted; + const reboundSubscription = { subscribed: true, onChainId: '15', lastReconciledOrdinal: 0 }; + agentLike.subscribedContextGraphs.set(contextGraphId, reboundSubscription); + releaseRoot(); + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }); + expect(bindSubscriptionOnChainId).not.toHaveBeenCalled(); + expect(persistContextGraphSubscriptionStrict).not.toHaveBeenCalled(); + expect(mockedMaterialize).not.toHaveBeenCalled(); + expect(syncCheckpoints).toEqual(new Map([['unchanged', 17]])); + expect(subscription).toEqual({ subscribed: true, onChainId: '14', lastReconciledOrdinal: 9 }); + expect(agentLike.subscribedContextGraphs.get(contextGraphId)).toBe(reboundSubscription); + }); + it('cancels and retries a hung root read within the graph deadline', async () => { vi.useFakeTimers(); const random = vi.spyOn(Math, 'random').mockReturnValue(0.5); diff --git a/packages/agent/test/e2e-memory-layers.test.ts b/packages/agent/test/e2e-memory-layers.test.ts index f85f1c1d3b..d04615d11a 100644 --- a/packages/agent/test/e2e-memory-layers.test.ts +++ b/packages/agent/test/e2e-memory-layers.test.ts @@ -1585,24 +1585,25 @@ describe('rootless graph-scoped KA lifecycle', () => { // assertion below (the CG would already be on-chain after the share). it('seals a FULL share on an UNregistered CG (no seal-time registration); registers + publishes at publish time', async () => { const agent = await createAgent('UnregisteredCgSealBot'); + const unregisteredCgId = `${CG_ID}-full-share-deferred-registration`; // LOCAL-ONLY CG: created but DELIBERATELY never registered on-chain. - await agent.createContextGraph({ id: CG_ID, name: 'Unregistered CG Seal E2E' }); + await agent.createContextGraph({ id: unregisteredCgId, name: 'Unregistered CG Seal E2E' }); const name = 'unregistered-cg-seal'; - await agent.assertion.create(CG_ID, name); - await agent.assertion.write(CG_ID, name, [ + await agent.assertion.create(unregisteredCgId, name); + await agent.assertion.write(unregisteredCgId, name, [ { subject: `${ENTITY_BASE}:ucs`, predicate: 'http://schema.org/name', object: '"Unregistered CG Seal"' }, ]); // Default FULL share — must SEAL despite the CG being unregistered (the seal // no longer depends on CG registration). - const fullShare = await agent.assertion.promote(CG_ID, name); + const fullShare = await agent.assertion.promote(unregisteredCgId, name); expect(fullShare.sealed).toBe(true); expect(fullShare.publishReady).toBe(true); // CORE ASSERTION: the CG is STILL unregistered after sealing — sealing did // NOT register it on-chain. Reintroducing seal-time registration breaks here. - const onChainIdAfterSeal = await agent.getContextGraphOnChainId(CG_ID); + const onChainIdAfterSeal = await agent.getContextGraphOnChainId(unregisteredCgId); expect(onChainIdAfterSeal == null).toBe(true); // And publishing the unregistered CG fails CLOSED for that exact reason — @@ -1612,7 +1613,7 @@ describe('rootless graph-scoped KA lifecycle', () => { // .code convention for SWM_SUBSET_NOT_SEALABLE / UNSEALED_SHARE_BLOCKED). let notRegisteredErr: any; try { - await agent.publishFromFinalizedAssertion(CG_ID, name); + await agent.publishFromFinalizedAssertion(unregisteredCgId, name); } catch (e) { notRegisteredErr = e; } @@ -1623,11 +1624,11 @@ describe('rootless graph-scoped KA lifecycle', () => { // Registration happens at PUBLISH time (the /vm/publish route's // ensureRegisteredForPublish step). After it, the same sealed asset publishes // to VM and confirms — no re-seal, no recreate. - await agent.ensureRegisteredForPublish(CG_ID); - const onChainIdAfterRegister = await agent.getContextGraphOnChainId(CG_ID); + await agent.ensureRegisteredForPublish(unregisteredCgId); + const onChainIdAfterRegister = await agent.getContextGraphOnChainId(unregisteredCgId); expect(onChainIdAfterRegister).toBeTruthy(); - const pub = await agent.publishFromFinalizedAssertion(CG_ID, name); + const pub = await agent.publishFromFinalizedAssertion(unregisteredCgId, name); expect(pub.status).toBe('confirmed'); expect(pub.ual).toBeDefined(); expect(pub.seal).toBeDefined(); diff --git a/packages/agent/test/exact-assets.test.ts b/packages/agent/test/exact-assets.test.ts index 58af41e7c9..467c47b634 100644 --- a/packages/agent/test/exact-assets.test.ts +++ b/packages/agent/test/exact-assets.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from 'vitest'; import { MAX_EXACT_SYNC_ASSETS, + MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + MAX_EXACT_SYNC_PHASE_QUADS_PER_ASSET, decodeExactAssetUals, encodeExactAssetUals, + exactAssetFilterKey, + exactSyncPhaseAccumulationLimits, normalizeExactAssetUals, } from '../src/sync/exact-assets.js'; @@ -11,9 +15,11 @@ const ual = (number: number) => describe('exact VM sync asset filter', () => { it('canonicalizes, deduplicates, and round-trips a bounded KA batch', () => { - const normalized = normalizeExactAssetUals([ual(1), ual(1), ual(2)]); + const normalized = normalizeExactAssetUals([ual(2), ual(1), ual(2)]); expect(normalized).toEqual([ual(1), ual(2)]); - expect(decodeExactAssetUals(encodeExactAssetUals(normalized!))).toEqual(normalized); + expect(decodeExactAssetUals(encodeExactAssetUals([ual(2), ual(1)]))).toEqual(normalized); + expect(exactAssetFilterKey([ual(2), ual(1)])) + .toBe(exactAssetFilterKey([ual(1), ual(2)])); }); it('fails closed for malformed or oversized present filters', () => { @@ -23,4 +29,11 @@ describe('exact VM sync asset filter', () => { (_, index) => ual(index), ))).toEqual([]); }); + + it('scales exact phase limits by the canonical asset count', () => { + expect(exactSyncPhaseAccumulationLimits([ual(2), ual(1), ual(2)])).toEqual({ + maxBytes: 2 * MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + maxQuads: 2 * MAX_EXACT_SYNC_PHASE_QUADS_PER_ASSET, + }); + }); }); diff --git a/packages/agent/test/outbox-shutdown-lifecycle.test.ts b/packages/agent/test/outbox-shutdown-lifecycle.test.ts index c283e5cd1b..80ef554320 100644 --- a/packages/agent/test/outbox-shutdown-lifecycle.test.ts +++ b/packages/agent/test/outbox-shutdown-lifecycle.test.ts @@ -3,9 +3,60 @@ import { DKGAgent } from '../src/dkg-agent.js'; import { DKGAgentBase } from '../src/dkg-agent-base.js'; import { VmReconcileDispatcher } from '../src/chain-reconciler.js'; import { FinalizationRuntime } from '../src/finalization-runtime.js'; -import { VmReconcileQueueClosedError } from '../src/vm-reconcile-service.js'; +import { + ContextGraphMembershipPersistScheduler, + ContextGraphMembershipPersistShutdownTimeoutError, +} from '../src/context-graph-membership-persist-scheduler.js'; +import { + VmReconcileQueueClosedError, + VmReconcileShutdownTimeoutError, +} from '../src/vm-reconcile-service.js'; describe('DKGAgent outbox shutdown lifecycle', () => { + it('closes and drains membership persistence before network and store teardown', async () => { + const membershipPersistence = new ContextGraphMembershipPersistScheduler(); + let releaseWrite!: () => void; + let markWriteStarted!: () => void; + const writeStarted = new Promise((resolve) => { markWriteStarted = resolve; }); + const writeGate = new Promise((resolve) => { releaseWrite = resolve; }); + const write = membershipPersistence.enqueue('cg\0node\0peer', async () => { + markWriteStarted(); + await writeGate; + }); + await writeStarted; + const closeStore = vi.fn(async () => {}); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + contextGraphMembershipPersistence: membershipPersistence, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain: vi.fn(async () => {}) }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + finalizationRuntime: new FinalizationRuntime(), + store: { close: closeStore }, + log: { warn: vi.fn() }, + }); + + const stopping = agent.stop(); + await Promise.resolve(); + expect(stopNode).not.toHaveBeenCalled(); + expect(closeStore).not.toHaveBeenCalled(); + + releaseWrite(); + await Promise.all([write, stopping]); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + }); + it('closes reconcile admission and cancels queued jobs before store teardown', async () => { let releaseActive!: () => void; let queuedStarted = false; @@ -23,7 +74,11 @@ describe('DKGAgent outbox shutdown lifecycle', () => { const agent = Object.create(DKGAgent.prototype) as any; Object.assign(agent, { started: true, - chainPoller: null, + chainPoller: { + stop: vi.fn(async () => { + expect(agent.vmReconcileRotationClosed).toBe(true); + }), + }, vmReconcileDispatcher: dispatcher, coreHostRecordingsClosed: false, drainCoreHostRecordings: vi.fn(async () => {}), @@ -55,15 +110,17 @@ describe('DKGAgent outbox shutdown lifecycle', () => { expect(closeStore).toHaveBeenCalledOnce(); }); - it('continues shutdown after the bounded reconcile drain timeout', async () => { + it('quarantines the instance after the bounded reconcile drain timeout', async () => { const originalTimeout = DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS; Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { configurable: true, value: 1, }); try { + let release!: () => void; + const activeGate = new Promise((resolve) => { release = resolve; }); const dispatcher = new VmReconcileDispatcher( - async () => new Promise(() => undefined), + async () => activeGate, () => undefined, ); void dispatcher.triggerManual('stuck'); @@ -87,20 +144,232 @@ describe('DKGAgent outbox shutdown lifecycle', () => { inFlightSubstrateFanOutCount: () => 0, router: { closePooling: vi.fn(async () => {}) }, node: { stop: stopNode }, + chain: { chainId: 'none' }, finalizationRuntime: new FinalizationRuntime(), store: { close: closeStore }, log: { warn }, }); - await expect(agent.stop()).resolves.toBeUndefined(); + await expect(agent.stop()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); - expect(stopNode).toHaveBeenCalledOnce(); - expect(closeStore).toHaveBeenCalledOnce(); + expect(stopNode).not.toHaveBeenCalled(); + expect(closeStore).not.toHaveBeenCalled(); expect(warn).toHaveBeenCalledWith( expect.anything(), - expect.stringContaining('1 VM reconcile job(s) still active after 1ms drain bound'), + expect.stringContaining('store/network teardown is blocked until stop() is retried'), + ); + + // Even after physical retirement, start remains blocked until a second + // stop finishes the deliberately incomplete teardown. + await expect(agent.start()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); + + release(); + await agent.vmReconcileRetirement; + await expect(agent.start()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); + await expect(agent.stop()).resolves.toBeUndefined(); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + expect(agent.started).toBe(false); + } finally { + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { + configurable: true, + value: originalTimeout, + }); + } + }); + + it('quarantines start and backing-store teardown after membership persistence times out', async () => { + const originalTimeout = DKGAgentBase.CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS; + Object.defineProperty(DKGAgentBase, 'CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS', { + configurable: true, + value: 1, + }); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + const membershipPersistence = new ContextGraphMembershipPersistScheduler(); + void membershipPersistence.enqueue('stuck', async () => gate); + await Promise.resolve(); + const closeStore = vi.fn(async () => {}); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + contextGraphMembershipPersistence: membershipPersistence, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain: vi.fn(async () => {}) }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + chain: { chainId: 'none' }, + finalizationRuntime: new FinalizationRuntime(), + store: { close: closeStore }, + log: { warn: vi.fn() }, + }); + + try { + await expect(agent.stop()).rejects.toBeInstanceOf( + ContextGraphMembershipPersistShutdownTimeoutError, + ); + expect(stopNode).not.toHaveBeenCalled(); + expect(closeStore).not.toHaveBeenCalled(); + await expect(agent.start()).rejects.toBeInstanceOf( + ContextGraphMembershipPersistShutdownTimeoutError, + ); + + release(); + await membershipPersistence.closeAndDrain(); + await expect(agent.start()).rejects.toBeInstanceOf( + ContextGraphMembershipPersistShutdownTimeoutError, ); + await expect(agent.stop()).resolves.toBeUndefined(); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + expect(agent.started).toBe(false); + } finally { + release(); + Object.defineProperty(DKGAgentBase, 'CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_MS', { + configurable: true, + value: originalTimeout, + }); + } + }); + + it('drains physically active VM reconciliation before closing the store', async () => { + const originalTimeout = DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { + configurable: true, + value: 1, + }); + let release!: () => void; + const physical = new Promise((resolve) => { release = resolve; }); + const closeStore = vi.fn(async () => {}); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + vmReconcilePhysicalRuns: new Set([physical]), + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain: vi.fn(async () => {}) }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + finalizationRuntime: new FinalizationRuntime(), + store: { close: closeStore }, + log: { warn: vi.fn() }, + }); + + try { + await expect(agent.stop()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); + expect(stopNode).not.toHaveBeenCalled(); + expect(closeStore).not.toHaveBeenCalled(); + + release(); + await agent.vmReconcileRetirement; + await agent.stop(); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + } finally { + release(); + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { + configurable: true, + value: originalTimeout, + }); + } + }); + + it('drains an entered ordinary graph-scoped commit before closing the store', async () => { + let release!: () => void; + const physicalCommit = new Promise((resolve) => { release = resolve; }); + const closeStore = vi.fn(async () => {}); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + graphScopedStorePhysicalRuns: new Set([physicalCommit]), + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain: vi.fn(async () => {}) }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + finalizationRuntime: new FinalizationRuntime(), + store: { close: closeStore }, + log: { warn: vi.fn() }, + }); + + const stopping = agent.stop(); + await Promise.resolve(); + expect(stopNode).not.toHaveBeenCalled(); + expect(closeStore).not.toHaveBeenCalled(); + + release(); + await stopping; + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + }); + + it('bounds chain-poller retirement before network and store teardown', async () => { + const originalTimeout = DKGAgentBase.VM_RECONCILE_SHUTDOWN_TIMEOUT_MS; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { + configurable: true, + value: 1, + }); + let release!: () => void; + const pollerDrain = new Promise((resolve) => { release = resolve; }); + const chainPoller = { stop: vi.fn(() => pollerDrain) }; + const closeStore = vi.fn(async () => {}); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain: vi.fn(async () => {}) }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + finalizationRuntime: new FinalizationRuntime(), + store: { close: closeStore }, + log: { warn: vi.fn() }, + }); + + try { + await expect(agent.stop()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); + expect(chainPoller.stop).toHaveBeenCalledOnce(); + expect(agent.chainPoller).toBe(chainPoller); + expect(stopNode).not.toHaveBeenCalled(); + expect(closeStore).not.toHaveBeenCalled(); + + release(); + await agent.vmReconcileRetirement; + expect(agent.chainPoller).toBeNull(); + await agent.stop(); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); } finally { + release(); Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { configurable: true, value: originalTimeout, diff --git a/packages/agent/test/p2p-peer-connect.test.ts b/packages/agent/test/p2p-peer-connect.test.ts index 651f711617..3348f7f14f 100644 --- a/packages/agent/test/p2p-peer-connect.test.ts +++ b/packages/agent/test/p2p-peer-connect.test.ts @@ -1,7 +1,12 @@ import { describe, it, expect } from 'vitest'; import { peerIdFromString } from '@libp2p/peer-id'; import { parseMultiaddrConnectTarget } from '../src/p2p/multiaddr-peer-target.js'; -import { connectToMultiaddr, primeCatchupConnections } from '../src/p2p/peer-connect.js'; +import { + connectToMultiaddr, + ensurePeerConnected, + primeCatchupConnections, +} from '../src/p2p/peer-connect.js'; +import { waitForPeerProtocol } from '../src/p2p/protocol-readiness.js'; function recorder(impl: (...args: A) => R) { const calls: A[] = []; @@ -128,3 +133,115 @@ describe('primeCatchupConnections', () => { expect(admissionCalls).toEqual([foreignPeer, eligiblePeer]); }); }); + +describe('abortable recovery connection helpers', () => { + it('forwards cancellation to the real direct-dial path', async () => { + const peerId = '12D3KooWQz2bQbQueABKRSjV9koF8VYsXk5TdCsUmPf5zAEZg3q6'; + const controller = new AbortController(); + let observedSignal: AbortSignal | undefined; + const dialStarted = Promise.withResolvers(); + const dial = (_peer: unknown, options?: { signal?: AbortSignal }) => { + observedSignal = options?.signal; + dialStarted.resolve(); + return new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => { + reject(new DOMException('dial aborted', 'AbortError')); + }, { once: true }); + }); + }; + let discoveryCalled = false; + + const connection = ensurePeerConnected({ + getConnections: () => [], + dial, + peerStore: { merge: async () => undefined }, + }, { + findAgentByPeerId: async () => { + discoveryCalled = true; + return undefined; + }, + } as any, peerId, { signal: controller.signal }); + + await dialStarted.promise; + controller.abort(); + await expect(connection).rejects.toMatchObject({ name: 'AbortError' }); + expect(observedSignal).toBe(controller.signal); + expect(discoveryCalled).toBe(false); + }); + + it('forwards cancellation through discovery after direct dial fails', async () => { + const peerId = '12D3KooWQz2bQbQueABKRSjV9koF8VYsXk5TdCsUmPf5zAEZg3q6'; + const controller = new AbortController(); + const discoveryStarted = Promise.withResolvers(); + let observedSignal: AbortSignal | undefined; + + const connection = ensurePeerConnected({ + getConnections: () => [], + dial: async () => { throw new Error('direct dial failed'); }, + peerStore: { merge: async () => undefined }, + }, { + findAgentByPeerId: async (_peerId: string, options?: { signal?: AbortSignal }) => { + observedSignal = options?.signal; + discoveryStarted.resolve(); + return new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => { + reject(new DOMException('discovery aborted', 'AbortError')); + }, { once: true }); + }); + }, + } as any, peerId, { signal: controller.signal }); + + await discoveryStarted.promise; + controller.abort(); + await expect(connection).rejects.toMatchObject({ name: 'AbortError' }); + expect(observedSignal).toBe(controller.signal); + }); + + it('forwards cancellation to the relay fallback dial', async () => { + const peerId = '12D3KooWQz2bQbQueABKRSjV9koF8VYsXk5TdCsUmPf5zAEZg3q6'; + const relayAddress = '/ip4/178.104.54.178/tcp/9090/p2p/12D3KooWSmU3owJvB9sFw8uApDgKrv2VBMecsGGvgAc4Gq6hB57M'; + const dialSignals: Array = []; + const merge = recorder(async () => undefined); + let dialAttempt = 0; + const dial = async (_peer: unknown, options?: { signal?: AbortSignal }) => { + dialSignals.push(options?.signal); + dialAttempt += 1; + if (dialAttempt === 1) throw new Error('direct dial failed'); + }; + + const controller = new AbortController(); + await ensurePeerConnected({ + getConnections: () => [], + dial, + peerStore: { merge }, + }, { + findAgentByPeerId: async () => ({ peerId, relayAddress }), + } as any, peerId, { signal: controller.signal }); + + expect(dialSignals).toEqual([controller.signal, controller.signal]); + expect(merge.calls).toHaveLength(1); + }); + + it('interrupts the real protocol-readiness delay', async () => { + const controller = new AbortController(); + let reads = 0; + const readiness = waitForPeerProtocol( + { + get: async () => { + reads += 1; + return { protocols: [] }; + }, + }, + { toString: () => 'peer-under-test' }, + '/dkg/test/sync', + 3, + 10_000, + controller.signal, + ); + + await Promise.resolve(); + controller.abort(); + await expect(readiness).rejects.toMatchObject({ name: 'AbortError' }); + expect(reads).toBe(1); + }); +}); diff --git a/packages/agent/test/private-cg-membership-bootstrap.test.ts b/packages/agent/test/private-cg-membership-bootstrap.test.ts index 276f52b333..7492ae4bd7 100644 --- a/packages/agent/test/private-cg-membership-bootstrap.test.ts +++ b/packages/agent/test/private-cg-membership-bootstrap.test.ts @@ -1581,5 +1581,67 @@ describe('private CG membership bootstrap recovery', () => { .get(member.address.toLowerCase())).toEqual([newPeerId]); expect((await agent.getContextGraphAllowedDelegateeKeys(contextGraphId)) .get(member.address.toLowerCase())).toEqual([newOpKey.toLowerCase()]); + }); + it('keeps join-approval snapshot and compensation inside the per-CG write lanes', async () => { + ({ agent } = await createAgent('PrivateBootstrapSerializedCompensation')); + const contextGraphId = 'private-bootstrap-serialized-compensation'; + const approvedAddress = agent.getDefaultAgentAddress()!; + const subscriptionRows = new Map(); + const membershipRows = new Map(); + let signalFirstSave!: () => void; + let releaseFirstSave!: () => void; + const firstSaveStarted = new Promise((resolve) => { signalFirstSave = resolve; }); + const firstSaveGate = new Promise((resolve) => { releaseFirstSave = resolve; }); + let saveCount = 0; + (agent as any).config.contextGraphSubscriptionStore = { + load: async (id: string) => subscriptionRows.get(id) ?? null, + loadAll: async () => [...subscriptionRows.values()], + save: async (record: any) => { + saveCount += 1; + if (saveCount === 1) { + signalFirstSave(); + await firstSaveGate; + } + subscriptionRows.set(record.id, { ...record }); + if (record.name === 'approval-B') throw new Error('approval B save failed after write'); + }, + delete: async (id: string) => { subscriptionRows.delete(id); }, + }; + const membershipKey = (record: any) => + `${record.contextGraphId}:${record.principalType}:${record.principalId.toLowerCase()}`; + (agent as any).config.contextGraphMembershipStore = { + loadAll: async () => [...membershipRows.values()], + upsert: async (record: any) => { membershipRows.set(membershipKey(record), { ...record }); }, + delete: async (cgId: string, principalType: string, principalId: string) => { + membershipRows.delete(`${cgId}:${principalType}:${principalId.toLowerCase()}`); + }, + }; + + const subscriptionA = { subscribed: true, name: 'ordinary-A' }; + (agent as any).subscribedContextGraphs.set(contextGraphId, subscriptionA); + const saveA = (agent as any).persistContextGraphSubscription(contextGraphId); + await firstSaveStarted; + + const subscriptionC = { subscribed: true, name: 'ordinary-C' }; + (agent as any).subscribedContextGraphs.set(contextGraphId, subscriptionC); + const saveC = (agent as any).persistContextGraphSubscription(contextGraphId); + const approvalB = (agent as any).persistJoinApprovalStateStrict( + contextGraphId, + { + contextGraphId, + principalType: 'agent', + principalId: approvedAddress, + role: 'participant', + status: 'active', + source: 'join-approved', + }, + { subscribed: true, name: 'approval-B' }, + ); + releaseFirstSave(); + + await expect(Promise.all([saveA, saveC])).resolves.toEqual([undefined, undefined]); + await expect(approvalB).rejects.toThrow('approval B save failed after write'); + expect(subscriptionRows.get(contextGraphId)).toMatchObject({ name: 'ordinary-C' }); + expect(membershipRows.size).toBe(0); }); }); diff --git a/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts b/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts index 14035787fc..af2499a8ae 100644 --- a/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts +++ b/packages/agent/test/rfc64-agent-inventory-lifecycle.test.ts @@ -15,6 +15,7 @@ import { ethers } from 'ethers'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { DKGAgent } from '../src/dkg-agent.js'; +import { ContextGraphMembershipPersistScheduler } from '../src/context-graph-membership-persist-scheduler.js'; import { FinalizationRuntime } from '../src/finalization-runtime.js'; import { INVENTORY_V1_RELATIVE_PATH, @@ -49,6 +50,7 @@ function syntheticAgent(dataDirectory?: string): any { const agent = Object.create(DKGAgent.prototype) as any; Object.assign(agent, { config: dataDirectory === undefined ? {} : { dataDir: dataDirectory }, + contextGraphMembershipPersistence: new ContextGraphMembershipPersistScheduler(), finalizationRuntime: new FinalizationRuntime(), rfc64PersistenceV1: undefined, }); diff --git a/packages/agent/test/rs-heal-stranded-kc.test.ts b/packages/agent/test/rs-heal-stranded-kc.test.ts index 998ffb89af..1e3873bf84 100644 --- a/packages/agent/test/rs-heal-stranded-kc.test.ts +++ b/packages/agent/test/rs-heal-stranded-kc.test.ts @@ -233,7 +233,9 @@ describe('healStrandedScopedKCs — content-binding gate', () => { lastReconciledOrdinal: 0, }, ]]); + agentLike.contextGraphBindingGenerations = new Map(); agentLike.reconcileCursors = new Map(); + agentLike.vmReconcilePhysicalRuns = new Set(); agentLike.vmReconcileDispatcher = { dispatch: async (_key: string, source: string): Promise => { priorities.push(source === 'periodic' ? 'background' : 'foreground'); @@ -289,7 +291,8 @@ describe('healStrandedScopedKCs — content-binding gate', () => { const countAfterFirst = await scopedTripleCount(store, TEST_CG, TEST_ONCHAIN); expect(countAfterFirst).toBeGreaterThan(0); - // Second run: the ASK-guard short-circuits (scoped now has batchId), so the + // Second run: the ASK-guard short-circuits (scoped now has batchId plus a + // materialization version), so the // scoped triple count is UNCHANGED — no re-copy, no duplication, no throw. await runHeal(store, TEST_CG, TEST_ONCHAIN); const countAfterSecond = await scopedTripleCount(store, TEST_CG, TEST_ONCHAIN); @@ -333,12 +336,82 @@ describe('healStrandedScopedKCs — content-binding gate', () => { const nsScopedMeta = contextGraphMetaUri(NS_CG, NS_ONCHAIN); expect(await readMaterializedVersion(store, nsScopedMeta, NS_UAL)).toEqual({ blockNumber: 0, txIndex: 0 }); - // Idempotent: a second run is a no-op (ASK-guard short-circuits on the now-present batchId). + // Idempotent: a second run is a no-op once both completion markers exist. const c1 = await scopedTripleCount(store, NS_CG, NS_ONCHAIN); await runHeal(store, NS_CG, NS_ONCHAIN); expect(await scopedTripleCount(store, NS_CG, NS_ONCHAIN)).toBe(c1); }); + it('repairs scoped metadata that has a batchId but no completion version', async () => { + const cg = 'partial-meta-cg'; + const onChainId = '23'; + const ual = 'did:dkg:hardhat:31337/0xpartial/42'; + await seedOntology(store, cg, onChainId); + await store.insert([ + ...metaQuads(ual, contextGraphMetaUri(cg)), + ...publicTriples().map((triple) => ({ ...triple, graph: contextGraphDataUri(cg) })), + // Simulate the old crash window: metadata including batchId committed, + // but the separate materializedVersion write never did. + ...metaQuads(ual, contextGraphMetaUri(cg, onChainId)), + ]); + expect(await readMaterializedVersion(store, contextGraphMetaUri(cg, onChainId), ual)).toBeNull(); + + await runHeal(store, cg, onChainId); + + expect(await readMaterializedVersion(store, contextGraphMetaUri(cg, onChainId), ual)) + .toEqual({ blockNumber: 0, txIndex: 0 }); + await expect(extractV10KCFromStore(store, BigInt(onChainId), KA_ID)).resolves.toBeTruthy(); + }); + + it('retries after a non-transactional endpoint partially copies metadata', async () => { + const cg = 'atomic-meta-retry-cg'; + const onChainId = '29'; + const ual = 'did:dkg:hardhat:31337/0xatomicretry/42'; + await seedOntology(store, cg, onChainId); + await store.insert([ + ...metaQuads(ual, contextGraphMetaUri(cg)), + ...publicTriples().map((triple) => ({ ...triple, graph: contextGraphDataUri(cg) })), + ]); + + const originalUpdate = store.update.bind(store); + const scopedMeta = contextGraphMetaUri(cg, onChainId); + let failMetadataCopyOnce = true; + store.update = async (sparql, options) => { + if ( + failMetadataCopyOnce + && sparql.includes(`FILTER(?p != <${DKG}materializedVersion>)`) + ) { + failMetadataCopyOnce = false; + await originalUpdate(sparql, options); + // Model a non-transactional endpoint that applied only part of the + // metadata INSERT before reporting failure. Completion must remain + // unstamped so the next sweep repairs the missing child row. + await originalUpdate( + `DELETE WHERE { + GRAPH <${scopedMeta}> { + <${ual}/1> <${RDF}type> ?type + } + }`, + options, + ); + throw new Error('injected partial metadata copy failure'); + } + return originalUpdate(sparql, options); + }; + + await runHeal(store, cg, onChainId); + expect(await readMaterializedVersion(store, scopedMeta, ual)).toBeNull(); + const missingChild = await store.query( + `ASK { GRAPH <${scopedMeta}> { <${ual}/1> <${RDF}type> ?type } }`, + ); + expect(missingChild.type === 'boolean' && missingChild.value).toBe(false); + + await runHeal(store, cg, onChainId); + expect(await readMaterializedVersion(store, scopedMeta, ual)) + .toEqual({ blockNumber: 0, txIndex: 0 }); + await expect(extractV10KCFromStore(store, BigInt(onChainId), KA_ID)).resolves.toBeTruthy(); + }); + it('relocates a publisher one-shot strand whose public data is in the VM graph only (read-both)', async () => { // The publisher's OWN one-shot publish() writes confirmed PUBLIC data to the // per-KA verifiable-memory graph `/_verifiable_memory//`, diff --git a/packages/agent/test/swm-curator-recovery-plan.test.ts b/packages/agent/test/swm-curator-recovery-plan.test.ts index d9948ab97e..f9808246a1 100644 --- a/packages/agent/test/swm-curator-recovery-plan.test.ts +++ b/packages/agent/test/swm-curator-recovery-plan.test.ts @@ -72,6 +72,128 @@ describe('private SWM curator recovery planning', () => { expect(plan.eligibleContextGraphIds).toEqual([]); }); + it('does not treat an empty registry after a failed metadata refresh as authoritative', async () => { + const curator = ethers.Wallet.createRandom().address.toLowerCase(); + const contextGraphId = `${curator}/refresh-failed-plan`; + const agent = await createAgent('CuratorRecoveryRefreshFailure'); + const internals = agent as unknown as { + localAgents: Map; + discovery: { findAgents: () => Promise> }; + refreshMetaFromCurator: (contextGraphId: string) => Promise; + resolveCuratorPeerIdsForCg: (contextGraphId: string) => Promise<{ + peerIds: string[]; + curatorIsLocal: boolean; + legacyTripleResolved: boolean; + lookupFailed?: boolean; + }>; + }; + internals.localAgents.clear(); + let lookups = 0; + internals.discovery = { + findAgents: async () => { + lookups += 1; + return []; + }, + }; + internals.refreshMetaFromCurator = async () => false; + + const result = await internals.resolveCuratorPeerIdsForCg(contextGraphId); + + expect(lookups).toBe(2); + expect(result).toEqual({ + peerIds: [], + curatorIsLocal: false, + legacyTripleResolved: false, + lookupFailed: true, + }); + }); + + it('stops structural curator discovery when its recovery target becomes stale', async () => { + const curator = ethers.Wallet.createRandom().address.toLowerCase(); + const contextGraphId = `${curator}/stale-curator-plan`; + const agent = await createAgent('CuratorRecoveryStaleTarget'); + const internals = agent as any; + internals.localAgents.clear(); + let releaseLookup!: () => void; + let markLookupStarted!: () => void; + const lookupStarted = new Promise((resolve) => { markLookupStarted = resolve; }); + const lookupGate = new Promise((resolve) => { releaseLookup = resolve; }); + let lookups = 0; + let refreshes = 0; + internals.discovery = { + findAgents: async () => { + lookups += 1; + markLookupStarted(); + await lookupGate; + return []; + }, + }; + internals.refreshMetaFromCurator = async () => { + refreshes += 1; + return false; + }; + let current = true; + const resolution = internals.resolveCuratorPeerIdsForCg(contextGraphId, { + isCurrent: () => current, + }); + + await lookupStarted; + current = false; + releaseLookup(); + + await expect(resolution).rejects.toMatchObject({ name: 'AbortError' }); + expect(lookups).toBe(1); + expect(refreshes).toBe(0); + }); + + it('walks an oversized structural-curator roster with an exclusive peer cursor', async () => { + const curator = ethers.Wallet.createRandom().address.toLowerCase(); + const contextGraphId = `${curator}/paged-curator-plan`; + const agent = await createAgent('CuratorRecoveryPagination'); + const pages = [ + ['peer-001', 'peer-002', 'peer-003'], + ['peer-002', 'peer-003', 'peer-004'], + ]; + const calls: Array<{ afterPeerId?: string; limit?: number }> = []; + const internals = agent as any; + internals.localAgents.clear(); + internals.discovery = { + findAgentPeerIdsByAddress: async ( + _address: string, + options: { afterPeerId?: string; limit?: number }, + ) => { + calls.push(options); + return pages[calls.length - 1] ?? []; + }, + findAgents: async () => [], + }; + + const first = await internals.resolveCuratorPeerIdsForCg(contextGraphId, { + maxPeerIds: 2, + pagePeerIds: 1, + }); + const second = await internals.resolveCuratorPeerIdsForCg(contextGraphId, { + maxPeerIds: 2, + pagePeerIds: 1, + afterPeerId: first.nextPageAfterPeerId, + }); + + expect(first).toMatchObject({ + peerIds: ['peer-001'], + overflowed: true, + nextPageAfterPeerId: 'peer-001', + }); + expect(second).toMatchObject({ + peerIds: ['peer-002'], + overflowed: true, + nextPageAfterPeerId: 'peer-002', + }); + expect(calls).toEqual([ + { limit: 3, signal: undefined }, + { afterPeerId: 'peer-001', limit: 2, signal: undefined }, + ]); + }); + async function createAgent(name: string): Promise { const agent = await DKGAgent.create({ name, diff --git a/packages/agent/test/sync-backpressure.test.ts b/packages/agent/test/sync-backpressure.test.ts index 382fe6b2a0..3f587d8382 100644 --- a/packages/agent/test/sync-backpressure.test.ts +++ b/packages/agent/test/sync-backpressure.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { backpressureRegistry, createOperationContext, + getMetrics, } from '@origintrail-official/dkg-core'; import { getSyncBackpressureSnapshot, @@ -11,11 +12,33 @@ import { SyncBackpressureBusyError, withGlobalSyncBackpressure, } from '../src/sync/backpressure.js'; -import { PriorityAdmissionQueue } from '../src/sync/priority-admission-queue.js'; +import { + PriorityAdmissionQueue, + type PriorityAdmissionAcquireOptions, +} from '../src/sync/priority-admission-queue.js'; import { LifecycleSyncMethods } from '../src/dkg-agent-lifecycle.js'; const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); +function queueOptions( + payload: string, + priority: number, + overrides: Partial> = {}, +): PriorityAdmissionAcquireOptions { + return { + payload, + ownerKey: payload, + lane: 'durable', + priority, + priorityClass: priority >= 2_000 ? 'elevated' : 'default', + queueLimit: 8, + agingThresholdMs: 10, + createBusyError: (reason) => new Error(reason), + createDisplacedError: () => new Error('displaced'), + ...overrides, + }; +} + describe('sync global backpressure', () => { it('preserves the original FIFO sequence across a running-to-queued handoff', async () => { let running = 0; @@ -217,6 +240,53 @@ describe('sync global backpressure', () => { ]); }); + it('starts foreground on the next release without preempting two active VM recoveries', async () => { + const ctx = createOperationContext('sync'); + const policy = resolveSyncGlobalBackpressure({ + syncGlobalMaxInflight: 2, + syncGlobalQueueLimit: 4, + }); + const events: string[] = []; + let releaseVmA!: () => void; + let releaseVmB!: () => void; + let releaseForeground!: () => void; + const blockingWork = (label: string, setRelease: (release: () => void) => void) => + async () => { + events.push(`${label}-start`); + await new Promise((resolve) => setRelease(resolve)); + events.push(`${label}-end`); + }; + + const vmA = withGlobalSyncBackpressure({ + policy, ctx, label: 'durable:vm-a', priority: 1_000, source: 'vm-recovery', + }, blockingWork('vm-a', (release) => { releaseVmA = release; })); + const vmB = withGlobalSyncBackpressure({ + policy, ctx, label: 'durable:vm-b', priority: 1_000, source: 'vm-recovery', + }, blockingWork('vm-b', (release) => { releaseVmB = release; })); + await tick(); + + const queuedVm = withGlobalSyncBackpressure({ + policy, ctx, label: 'durable:vm-c', priority: 1_000, source: 'vm-recovery', + }, async () => { events.push('vm-c-start'); }); + const foreground = withGlobalSyncBackpressure({ + policy, ctx, label: 'durable:foreground', priority: 2_000, + source: 'catchup-foreground', + }, blockingWork('foreground', (release) => { releaseForeground = release; })); + await tick(); + + expect(events).toEqual(['vm-a-start', 'vm-b-start']); + releaseVmA(); + await tick(); + expect(events).toEqual(['vm-a-start', 'vm-b-start', 'vm-a-end', 'foreground-start']); + + releaseForeground(); + await foreground; + await tick(); + expect(events).toContain('vm-c-start'); + releaseVmB(); + await Promise.all([vmA, vmB, queuedVm]); + }); + it('carries the admission source from the production helper through to the scheduler', async () => { // The two halves of this contract were covered separately: the call sites // were proven to SUPPLY a source, and `withGlobalSyncBackpressure` was proven @@ -380,29 +450,534 @@ describe('sync global backpressure', () => { expect(events).toEqual(['running', 'high', 'low']); }); - it('runs the oldest aged entry before newer elevated work', async () => { + it('bounds an aged lower-priority entry behind one raw-priority overtake', async () => { const ctx = createOperationContext('sync'); const policy = resolveSyncGlobalBackpressure({ syncGlobalMaxInflight: 1, syncGlobalQueueLimit: 3 }); const events: string[] = []; - let now = 0; let unblock!: () => void; - const running = withGlobalSyncBackpressure({ policy, ctx, label: 'running', now: () => now }, async () => { + const running = withGlobalSyncBackpressure({ policy, ctx, label: 'running' }, async () => { await new Promise((resolve) => { unblock = resolve; }); }); await tick(); const agedLow = withGlobalSyncBackpressure({ policy, ctx, label: 'aged-low', priority: -1, priorityClass: 'deprioritized', - agingThresholdMs: 10, now: () => now, + agingThresholdMs: 0, }, async () => { events.push('aged-low'); }); - now = 5; const high = withGlobalSyncBackpressure({ policy, ctx, label: 'high', priority: 100, priorityClass: 'elevated', - agingThresholdMs: 10, now: () => now, + agingThresholdMs: 0, }, async () => { events.push('high'); }); - now = 11; unblock(); await Promise.all([running, agedLow, high]); - expect(events).toEqual(['aged-low', 'high']); + expect(events).toEqual(['high', 'aged-low']); + }); + + it('uses raw numeric priority and pays one aged-service debt after an overtake', async () => { + let now = 0; + let running = 0; + let enabled = false; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => enabled && running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const low = queue.acquire(queueOptions('low-1000', 1_000, { + priorityClass: 'elevated', + })); + const high = queue.acquire(queueOptions('high-2000', 2_000, { + priorityClass: 'deprioritized', + })); + now = 20; + enabled = true; + queue.pump(); + + const releaseHigh = await high.release; + expect(starts).toEqual(['high-2000']); + releaseHigh(); + const releaseLow = await low.release; + expect(starts).toEqual(['high-2000', 'low-1000']); + releaseLow(); + }); + + it('does not let a newly arriving higher maximum rearm an existing debt', async () => { + let now = 0; + let running = 0; + let enabled = false; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => enabled && running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const low = queue.acquire(queueOptions('aged-1000', 1_000)); + const firstHigh = queue.acquire(queueOptions('high-2000', 2_000)); + now = 20; + enabled = true; + queue.pump(); + const releaseFirstHigh = await firstHigh.release; + const laterMaximum = queue.acquire(queueOptions('later-3000', 3_000)); + + releaseFirstHigh(); + const releaseLow = await low.release; + expect(starts).toEqual(['high-2000', 'aged-1000']); + releaseLow(); + const releaseLaterMaximum = await laterMaximum.release; + expect(starts).toEqual(['high-2000', 'aged-1000', 'later-3000']); + releaseLaterMaximum(); + }); + + it('keeps debt while its aged recipient is temporarily not runnable', async () => { + let now = 0; + let running = 0; + let enabled = false; + let lowBlocked = true; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: (entry) => ( + enabled + && running < 1 + && (entry.payload !== 'aged-low' || !lowBlocked) + ), + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const low = queue.acquire(queueOptions('aged-low', 1_000)); + const firstHigh = queue.acquire(queueOptions('high-1', 2_000)); + now = 20; + enabled = true; + queue.pump(); + const releaseFirstHigh = await firstHigh.release; + const secondHigh = queue.acquire(queueOptions('high-2', 2_000)); + + releaseFirstHigh(); + const releaseSecondHigh = await secondHigh.release; + expect(starts).toEqual(['high-1', 'high-2']); + releaseSecondHigh(); + lowBlocked = false; + queue.pump(); + const releaseLow = await low.release; + expect(starts).toEqual(['high-1', 'high-2', 'aged-low']); + releaseLow(); + }); + + it('fills two free slots with one high overtake and the owed aged turn', async () => { + let now = 0; + let running = 0; + let enabled = false; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => enabled && running < 2, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const low1 = queue.acquire(queueOptions('low-1', 1_000)); + const low2 = queue.acquire(queueOptions('low-2', 1_000)); + const high = queue.acquire(queueOptions('high', 2_000)); + now = 20; + enabled = true; + queue.pump(); + + const releaseHigh = await high.release; + const releaseLow1 = await low1.release; + expect(running).toBe(2); + expect(starts).toEqual(['high', 'low-1']); + releaseHigh(); + const releaseLow2 = await low2.release; + expect(starts).toEqual(['high', 'low-1', 'low-2']); + releaseLow1(); + releaseLow2(); + }); + + it('protects one oldest aged lower-priority entry from a displacement flood', async () => { + let now = 0; + let running = 0; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + const atCapacity = { queueLimit: 4 }; + + const blocker = queue.acquire(queueOptions('blocker', 0, atCapacity)); + const releaseBlocker = await blocker.release; + const oldest = queue.acquire(queueOptions('oldest-aged', 0, atCapacity)); + const replaceable = queue.acquire(queueOptions('replaceable-aged', 0, atCapacity)); + const medium = queue.acquire(queueOptions('medium', 5, atCapacity)); + const upper = queue.acquire(queueOptions('upper', 6, atCapacity)); + const displaced = replaceable.release.catch((error: unknown) => error); + now = 20; + const high = queue.acquire(queueOptions('high', 10, atCapacity)); + + expect(await displaced).toMatchObject({ message: 'displaced' }); + expect(queue.entries().map((entry) => entry.payload)).toContain('oldest-aged'); + expect(queue.entries().map((entry) => entry.payload)).not.toContain('replaceable-aged'); + + releaseBlocker(); + const releaseHigh = await high.release; + releaseHigh(); + const releaseOldest = await oldest.release; + releaseOldest(); + const releaseUpper = await upper.release; + releaseUpper(); + const releaseMedium = await medium.release; + releaseMedium(); + expect(starts).toEqual(['blocker', 'high', 'oldest-aged', 'upper', 'medium']); + }); + + it('protects the sole aged entry when the queue limit is one', async () => { + let now = 0; + let running = 0; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + const oneSlot = { queueLimit: 1 }; + + const blocker = queue.acquire(queueOptions('blocker', 0, oneSlot)); + const releaseBlocker = await blocker.release; + const aged = queue.acquire(queueOptions('aged', 0, oneSlot)); + now = 20; + + expect(() => queue.acquire(queueOptions('high', 10, oneSlot))).toThrow('global_queue_full'); + expect(queue.entries().map((entry) => entry.payload)).toEqual(['aged']); + + releaseBlocker(); + const releaseAged = await aged.release; + releaseAged(); + expect(starts).toEqual(['blocker', 'aged']); + }); + + it('clears debt when its last aged recipient is cancelled', async () => { + let now = 0; + let running = 0; + let enabled = false; + const starts: string[] = []; + const controller = new AbortController(); + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => enabled && running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const cancelled = queue.acquire(queueOptions('cancelled-low', 1_000, { + signal: controller.signal, + })); + const firstHigh = queue.acquire(queueOptions('first-high', 2_000)); + now = 20; + enabled = true; + queue.pump(); + const releaseFirstHigh = await firstHigh.release; + controller.abort(new Error('cancelled')); + await expect(cancelled.release).rejects.toThrow('cancelled'); + + const nextLow = queue.acquire(queueOptions('next-low', 1_000)); + const nextHigh = queue.acquire(queueOptions('next-high', 2_000)); + now = 40; + releaseFirstHigh(); + const releaseNextHigh = await nextHigh.release; + expect(starts).toEqual(['first-high', 'next-high']); + releaseNextHigh(); + const releaseNextLow = await nextLow.release; + releaseNextLow(); + }); + + it('clears debt when its last aged recipient times out', async () => { + let now = 0; + let running = 0; + let enabled = false; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => enabled && running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const timedOut = queue.acquire(queueOptions('timed-out-low', 1_000, { + timeoutMs: 5, + createTimeoutError: () => new Error('timed out'), + })); + const timeoutResult = timedOut.release.catch((error: unknown) => error); + const firstHigh = queue.acquire(queueOptions('first-high', 2_000)); + now = 20; + enabled = true; + queue.pump(); + const releaseFirstHigh = await firstHigh.release; + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(await timeoutResult).toMatchObject({ message: 'timed out' }); + + const nextLow = queue.acquire(queueOptions('next-low', 1_000)); + const nextHigh = queue.acquire(queueOptions('next-high', 2_000)); + now = 40; + releaseFirstHigh(); + const releaseNextHigh = await nextHigh.release; + expect(starts).toEqual(['first-high', 'next-high']); + releaseNextHigh(); + const releaseNextLow = await nextLow.release; + releaseNextLow(); + }); + + it('records a queued timeout as a rejected scheduler decision', async () => { + let running = 0; + const decisionCounter = getMetrics().syncSchedulerDecisionsTotal as unknown as { + add(value: number, attributes: Record): void; + }; + const originalAdd = decisionCounter.add; + const decisions: Array> = []; + decisionCounter.add = (_value, attributes) => { decisions.push(attributes); }; + try { + const queue = new PriorityAdmissionQueue({ + canRun: () => running < 1, + onStart: () => { + running += 1; + return () => { running -= 1; }; + }, + }); + const active = queue.acquire(queueOptions('active', 0)); + const releaseActive = await active.release; + const timedOut = queue.acquire(queueOptions('timed-out', 0, { + timeoutMs: 5, + createTimeoutError: () => new Error('timed out'), + })); + + await expect(timedOut.release).rejects.toThrow('timed out'); + expect(decisions).toContainEqual({ + lane: 'durable', priority_class: 'default', outcome: 'rejected', + }); + releaseActive(); + } finally { + decisionCounter.add = originalAdd; + } + }); + + it('preserves debt across a running-to-queued responder handoff', async () => { + let now = 0; + let running = 0; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => running < 1, + onStart: (entry) => { + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const first = queue.acquire(queueOptions('first-stage', 0, { + ownerKey: 'peer-a', + queueLimit: 4, + ownerQueueLimit: 4, + reserveForHandoff: true, + })); + const releaseFirst = await first.release; + const low = queue.acquire(queueOptions('aged-low', 1_000, { + ownerKey: 'peer-b', + queueLimit: 4, + ownerQueueLimit: 4, + })); + now = 20; + const high = queue.acquire(queueOptions('high-2000', 2_000, { + ownerKey: 'peer-c', + queueLimit: 4, + ownerQueueLimit: 4, + })); + const handoff = first.handoff!({ + payload: 'handoff-3000', + lane: 'responder', + priority: 3_000, + priorityClass: 'deprioritized', + agingThresholdMs: 10, + createBusyError: (reason) => new Error(reason), + createDisplacedError: () => new Error('displaced'), + }); + + releaseFirst(); + const releaseHandoff = await handoff.release; + releaseHandoff(); + const releaseLow = await low.release; + expect(starts).toEqual(['first-stage', 'handoff-3000', 'aged-low']); + releaseLow(); + const releaseHigh = await high.release; + releaseHigh(); + }); + + it('creates fairness debt only after onStart succeeds', async () => { + let now = 0; + let running = 0; + let enabled = false; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: () => enabled && running < 1, + onStart: (entry) => { + if (entry.payload === 'failing-3000') throw new Error('start failed'); + running += 1; + starts.push(entry.payload); + return () => { running -= 1; }; + }, + }); + + const low = queue.acquire(queueOptions('aged-1000', 1_000)); + const medium = queue.acquire(queueOptions('medium-2000', 2_000)); + const failing = queue.acquire(queueOptions('failing-3000', 3_000)); + const failed = failing.release.catch((error: unknown) => error); + now = 20; + enabled = true; + queue.pump(); + + expect(await failed).toMatchObject({ message: 'start failed' }); + const releaseMedium = await medium.release; + expect(starts).toEqual(['medium-2000']); + releaseMedium(); + const releaseLow = await low.release; + releaseLow(); + }); + + it('uses genuinely free capacity when older work is blocked by its own lower limit', async () => { + type MixedPolicyPayload = { name: string; inflightLimit: number }; + let now = 0; + let running = 0; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + now: () => now, + canRun: (entry) => running < entry.payload.inflightLimit, + onStart: (entry) => { + running += 1; + starts.push(entry.payload.name); + return () => { running -= 1; }; + }, + }); + const options = (name: string, inflightLimit: number, priority: number) => ({ + payload: { name, inflightLimit }, + ownerKey: name, + lane: 'durable' as const, + priority, + priorityClass: priority > 0 ? 'elevated' as const : 'default' as const, + queueLimit: 1, + agingThresholdMs: 10, + createBusyError: (reason: string) => new Error(reason), + createDisplacedError: () => new Error('displaced'), + }); + + const blocker = queue.acquire(options('limit-one-blocker', 1, 0)); + const releaseBlocker = await blocker.release; + const aged = queue.acquire(options('aged-limit-one', 1, 0)); + now = 20; + + const higherCapacity = queue.acquire(options('limit-two-foreground', 2, 1)); + expect(higherCapacity.status).toBe('running'); + const releaseHigherCapacity = await higherCapacity.release; + expect(starts).toEqual(['limit-one-blocker', 'limit-two-foreground']); + expect(aged.status).toBe('queued'); + + releaseHigherCapacity(); + releaseBlocker(); + const releaseAged = await aged.release; + expect(starts).toEqual([ + 'limit-one-blocker', + 'limit-two-foreground', + 'aged-limit-one', + ]); + releaseAged(); + }); + + it('rolls back direct-start capacity when onStart throws after claiming it', async () => { + let running = 0; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + canRun: () => running < 1, + onStart: (entry) => { + running += 1; + if (entry.payload === 'failing') throw new Error('start failed after claim'); + starts.push(entry.payload); + return () => { running -= 1; }; + }, + onStartFailureRollback: () => { running -= 1; }, + }); + + expect(() => queue.acquire(queueOptions('failing', 0))).toThrow( + 'start failed after claim', + ); + expect(running).toBe(0); + + const subsequent = queue.acquire(queueOptions('subsequent', 0)); + const releaseSubsequent = await subsequent.release; + expect(subsequent.status).toBe('running'); + expect(starts).toEqual(['subsequent']); + expect(running).toBe(1); + releaseSubsequent(); + expect(running).toBe(0); + }); + + it('rolls back queued-start capacity and starts the next waiter in the same pump', async () => { + let running = 0; + const starts: string[] = []; + const queue = new PriorityAdmissionQueue({ + canRun: () => running < 1, + onStart: (entry) => { + running += 1; + if (entry.payload === 'failing') throw new Error('queued start failed after claim'); + starts.push(entry.payload); + return () => { running -= 1; }; + }, + onStartFailureRollback: () => { running -= 1; }, + }); + + const blocker = queue.acquire(queueOptions('blocker', 0)); + const releaseBlocker = await blocker.release; + const failing = queue.acquire(queueOptions('failing', 10)); + const failingResult = failing.release.catch((error: unknown) => error); + const subsequent = queue.acquire(queueOptions('subsequent', 0)); + + releaseBlocker(); + + expect(await failingResult).toMatchObject({ message: 'queued start failed after claim' }); + const releaseSubsequent = await subsequent.release; + expect(starts).toEqual(['blocker', 'subsequent']); + expect(running).toBe(1); + expect(queue.length).toBe(0); + releaseSubsequent(); + expect(running).toBe(0); }); it('displaces only strictly lower-priority queued work when the queue is full', async () => { @@ -639,6 +1214,7 @@ describe('sync global backpressure', () => { let running = 0; let now = 1_000; const queue = new PriorityAdmissionQueue({ + now: () => now, canRun: () => running < 1, onStart: () => { running += 1; @@ -649,7 +1225,6 @@ describe('sync global backpressure', () => { operation: (entry) => entry.payload, inflightLimit: () => 1, thresholds: { degradedQueueAgeMs: 5_000 }, - now: () => now, }, }); const options = (payload: string) => ({ @@ -659,7 +1234,6 @@ describe('sync global backpressure', () => { priorityClass: 'default' as const, queueLimit: 2, agingThresholdMs: 30_000, - now: () => now, createBusyError: () => new Error('full'), createDisplacedError: () => new Error('displaced'), }); @@ -687,6 +1261,7 @@ describe('sync global backpressure', () => { }], }], }); + expect(queue.oldestAgeMs()).toBe(6_000); releaseFirst(); const releaseSecond = await second.release; diff --git a/packages/agent/test/sync-exact-accumulation.test.ts b/packages/agent/test/sync-exact-accumulation.test.ts new file mode 100644 index 0000000000..ef13f0f250 --- /dev/null +++ b/packages/agent/test/sync-exact-accumulation.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from 'vitest'; +import { + exactAssetFilterKey, + MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, +} from '../src/sync/exact-assets.js'; +import { + getSyncCheckpointKey, + MemorySyncCheckpointStore, +} from '../src/sync/checkpoint/state.js'; +import { fetchSyncPages } from '../src/sync/requester/page-fetch.js'; + +const EXACT_UAL = 'did:dkg:base:84532/0x1111111111111111111111111111111111111111/7'; +const encoder = new TextEncoder(); +type FetchParams = Parameters[0]; + +function fetchParams(overrides: Partial = {}): FetchParams { + return { + ctx: { operationId: 'test', operationName: 'sync' }, + remotePeerId: 'legacy-peer', + contextGraphId: 'large-legacy-cg', + includeSharedMemory: false, + phase: 'data', + graphUri: 'urn:data', + deadline: Date.now() + 10_000, + syncPageTimeoutMs: 1_000, + syncRouterAttempts: 1, + syncPageRetryAttempts: 1, + syncPageSize: 8_192, + syncDeniedResponse: 'denied', + debugSyncProgress: false, + protocolSync: '/dkg/test/sync', + checkpointStore: new MemorySyncCheckpointStore(), + assetUals: [EXACT_UAL], + maxAcceptedBytes: 1_000, + maxAcceptedQuads: 100, + buildSyncRequest: async () => encoder.encode('request'), + parseAndFilter: async () => ({ quads: [], totalQuads: 0 }), + send: async () => new Uint8Array(), + logWarn: () => {}, + logInfo: () => {}, + logDebug: () => {}, + ...overrides, + }; +} + +describe('exact sync accumulation limits', () => { + it('rejects excess wire bytes before parsing and clears resumable state', async () => { + const firstPage = encoder.encode('page-one'); + const secondPage = encoder.encode('page-two'); + const checkpointStore = new MemorySyncCheckpointStore(); + const checkpointKey = getSyncCheckpointKey( + 'legacy-peer', + 'large-legacy-cg', + false, + 'data', + undefined, + undefined, + undefined, + exactAssetFilterKey([EXACT_UAL]), + ); + checkpointStore.set(checkpointKey, 7); + checkpointStore.setResponderSession?.( + checkpointKey, + 'legacy-session', + Date.now() + 60_000, + ); + const requestedOffsets: number[] = []; + let sends = 0; + let parses = 0; + + await expect(fetchSyncPages(fetchParams({ + checkpointStore, + maxAcceptedBytes: firstPage.byteLength, + buildSyncRequest: async (_contextGraphId, offset) => { + requestedOffsets.push(offset); + return encoder.encode('request'); + }, + parseAndFilter: async () => { + parses += 1; + return { + quads: [{ subject: 'urn:s', predicate: 'urn:p', object: '"o"', graph: 'urn:data' }], + totalQuads: 1, + }; + }, + send: async () => { + sends += 1; + return sends === 1 ? firstPage : secondPage; + }, + }))).rejects.toMatchObject({ + code: 'SYNC_PAGE_ACCUMULATION_LIMIT', + dimension: 'bytes', + actual: firstPage.byteLength + secondPage.byteLength, + limit: firstPage.byteLength, + }); + + expect(requestedOffsets[0]).toBe(7); + expect(sends).toBe(2); + expect(parses).toBe(1); + expect(checkpointStore.get(checkpointKey)).toBeUndefined(); + + const retry = await fetchSyncPages(fetchParams({ + checkpointStore, + send: async () => new Uint8Array(), + })); + expect(retry.resumedFromOffset).toBe(0); + expect(retry.responderSessionStartedFresh).toBe(true); + }); + + it('rejects excess parsed quads before retaining another legacy page', async () => { + let sends = 0; + let parses = 0; + + await expect(fetchSyncPages(fetchParams({ + phase: 'meta', + graphUri: 'urn:meta', + maxAcceptedQuads: 1, + parseAndFilter: async () => { + parses += 1; + return { + quads: [{ subject: `urn:s:${parses}`, predicate: 'urn:p', object: '"o"', graph: 'urn:meta' }], + totalQuads: 1, + }; + }, + send: async () => { + sends += 1; + return encoder.encode(`page-${sends}`); + }, + }))).rejects.toMatchObject({ + code: 'SYNC_PAGE_ACCUMULATION_LIMIT', + dimension: 'quads', + actual: 2, + limit: 1, + }); + + expect(sends).toBe(2); + expect(parses).toBe(2); + }); + + it('accepts the exact 4 MiB per-asset boundary and rejects one byte more', async () => { + const boundaryPage = new Uint8Array(MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET).fill(97); + let equalitySends = 0; + const result = await fetchSyncPages(fetchParams({ + remotePeerId: 'compatible-peer', + contextGraphId: 'bounded-exact-cg', + maxAcceptedBytes: MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + maxAcceptedQuads: 1, + parseAndFilter: async () => ({ + quads: [{ subject: 'urn:s', predicate: 'urn:p', object: '"o"', graph: 'urn:data' }], + totalQuads: 1, + }), + send: async () => { + equalitySends += 1; + return equalitySends === 1 ? boundaryPage : new Uint8Array(); + }, + })); + + let overBoundaryParsed = false; + await expect(fetchSyncPages(fetchParams({ + remotePeerId: 'over-boundary-peer', + contextGraphId: 'bounded-exact-cg', + maxAcceptedBytes: MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + maxAcceptedQuads: 1, + parseAndFilter: async () => { + overBoundaryParsed = true; + return { quads: [], totalQuads: 0 }; + }, + send: async () => new Uint8Array(MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET + 1), + }))).rejects.toMatchObject({ + code: 'SYNC_PAGE_ACCUMULATION_LIMIT', + dimension: 'bytes', + actual: MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET + 1, + limit: MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + }); + + expect(result.completed).toBe(true); + expect(result.quads).toHaveLength(1); + expect(result.bytesReceived).toBe(MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET); + expect(equalitySends).toBe(2); + expect(overBoundaryParsed).toBe(false); + }); +}); diff --git a/packages/agent/test/sync-fetch-coalescing.test.ts b/packages/agent/test/sync-fetch-coalescing.test.ts index beb7a8840b..eac3085634 100644 --- a/packages/agent/test/sync-fetch-coalescing.test.ts +++ b/packages/agent/test/sync-fetch-coalescing.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { createOperationContext, PROTOCOL_SYNC, @@ -8,9 +8,15 @@ import { DKGAgent, FOREGROUND_CATCHUP_SYNC_PRIORITY, } from '../src/index.js'; +import type { ContextGraphMembershipStore } from '../src/dkg-agent-types.js'; import { resolveSyncGlobalBackpressure, SyncBackpressureBusyError, withGlobalSyncBackpressure } from '../src/sync/backpressure.js'; import type { SyncPhase } from '../src/sync/auth/request-build.js'; import type { SyncPageResult } from '../src/sync/requester/page-fetch.js'; +import { DKGAgentBase } from '../src/dkg-agent-base.js'; +import { + VmReconcileQueueClosedError, + VmReconcileShutdownTimeoutError, +} from '../src/vm-reconcile-service.js'; import { stubLifecycleFetch } from './_helpers/sync-fetch-coalescing.js'; const PEER_A = '12D3KooWSmU3owJvB9sFw8uApDgKrv2VBMecsGGvgAc4Gq6hB57M'; @@ -105,6 +111,7 @@ function emptySyncPage(phase: string): SyncPageResult { quads: [], bytesReceived: 0, resumedFromOffset: 0, + responderSessionStartedFresh: true, nextOffset: 0, checkpointKey: `checkpoint:${phase}`, completed: true, @@ -119,11 +126,13 @@ async function createAgentWithSend( syncGlobalQueueLimit: number; syncContextGraphPriorities?: Record; }, + contextGraphMembershipStore?: ContextGraphMembershipStore, ): Promise { const agent = await DKGAgent.create({ name: 'SyncFetchCoalescing', listenHost: '127.0.0.1', chainAdapter: new MockChainAdapter(), + contextGraphMembershipStore, ...backpressure, }); (agent as any).messenger = { sendToPeer }; @@ -131,6 +140,96 @@ async function createAgentWithSend( return agent; } +describe('exact VM recovery lifecycle', () => { + it('reopens reconcile and membership persistence admission on same-object restart', async () => { + const membershipUpsert = vi.fn(async () => undefined); + const agent = await createAgentWithSend( + async () => new Uint8Array(0), + undefined, + { + loadAll: async () => [], + upsert: membershipUpsert, + delete: async () => undefined, + }, + ); + try { + await agent.start(); + const initialGeneration = (agent as any).vmReconcileLifecycleGeneration; + await agent.stop(); + expect((agent as any).vmReconcileRotationClosed).toBe(true); + expect((agent as any).vmReconcileLifecycleGeneration).toBe(initialGeneration + 1); + expect((agent as any).contextGraphMembershipPersistence.status().closed).toBe(true); + + await agent.start(); + expect((agent as any).vmReconcileRotationClosed).toBe(false); + expect((agent as any).vmReconcileLifecycleGeneration).toBe(initialGeneration + 1); + expect((agent as any).vmReconcileDispatcher.snapshot().closed).toBe(false); + expect((agent as any).contextGraphMembershipPersistence.status().closed).toBe(false); + const upsertsBeforeRestartProbe = membershipUpsert.mock.calls.length; + await agent.upsertContextGraphMember({ + contextGraphId: 'restart-membership', + principalType: 'node', + principalId: PEER_A, + status: 'active', + }, { strict: true }); + expect(membershipUpsert).toHaveBeenCalledTimes(upsertsBeforeRestartProbe + 1); + } finally { + await agent.stop().catch(() => {}); + } + }); + + it('quarantines a physically active reconcile until shutdown is retried', async () => { + const timeoutDescriptor = Object.getOwnPropertyDescriptor( + DKGAgentBase, + 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', + )!; + Object.defineProperty(DKGAgentBase, 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', { + ...timeoutDescriptor, + value: 1, + }); + const agent = await createAgentWithSend(async () => new Uint8Array(0)); + const targetEntered = deferred(); + const releaseTarget = deferred(); + const heal = vi.fn(async () => undefined); + try { + await agent.start(); + const oldDispatcher = (agent as any).vmReconcileDispatcher; + (agent as any).resolveVmReconcileTarget = async () => { + targetEntered.resolve(); + await releaseTarget.promise; + return {}; + }; + (agent as any).healStrandedScopedKCs = heal; + + const oldRun = (agent as any).runVmReconcileForCg('restart-retirement', 'manual'); + const oldOutcome = oldRun.catch((error: unknown) => error); + await targetEntered.promise; + await expect(agent.stop()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); + expect(oldDispatcher.snapshot()).toMatchObject({ active: 0, closed: true }); + await expect(oldOutcome).resolves.toBeInstanceOf(VmReconcileQueueClosedError); + await expect(agent.start()).rejects.toBeInstanceOf(VmReconcileShutdownTimeoutError); + + releaseTarget.resolve(); + await (agent as any).vmReconcileRetirement; + expect(heal).not.toHaveBeenCalled(); + await agent.stop(); + + await agent.start(); + const newDispatcher = (agent as any).vmReconcileDispatcher; + expect(newDispatcher).not.toBe(oldDispatcher); + expect(newDispatcher.snapshot().closed).toBe(false); + } finally { + releaseTarget.resolve(); + await agent.stop().catch(() => {}); + Object.defineProperty( + DKGAgentBase, + 'VM_RECONCILE_SHUTDOWN_TIMEOUT_MS', + timeoutDescriptor, + ); + } + }); +}); + function fetchPages(agent: DKGAgent, args: FetchArgs = {}): Promise { return (agent as any).fetchSyncPages( createOperationContext('sync'), @@ -465,6 +564,85 @@ describe('DKGAgent sync fetch coalescing', () => { } }); + it('shares one physical exact outcome across public and detailed joiners', async () => { + const firstMetaFetch = deferred(); + let fetchCalls = 0; + const agent = await createAgentWithSend(async () => new Uint8Array(0)); + stubLifecycleFetch(agent, async ({ phase }) => { + fetchCalls++; + if (fetchCalls === 1) return firstMetaFetch.promise; + return emptySyncPage(phase); + }); + (agent as any).processDurableBatchInWorker = async () => ({ + verifiedData: [], + verifiedMeta: [], + consumedUnpersistedMetaTriples: 0, + totalFetchedDataQuads: 0, + totalFetchedMetaQuads: 0, + rejectedKcs: 0, + emptyResponses: 1, + metaOnlyResponses: 0, + verifiedPrivateOnlyResponses: 0, + dataRejectedMissingMeta: 0, + }); + + try { + const publicResult = (agent as any).syncExactKnowledgeAssetsFromPeer( + PEER_A, + 'coalesced-cg', + [EXACT_UAL_7], + ); + await waitFor(() => fetchCalls === 1); + const detailedResult = (agent as any).syncExactKnowledgeAssetsFromPeerDetailed( + PEER_A, + 'coalesced-cg', + [EXACT_UAL_7], + ); + firstMetaFetch.resolve(emptySyncPage('meta')); + + const [projected, detailed] = await Promise.all([publicResult, detailedResult]); + expect(fetchCalls).toBe(2); + expect(projected).toBe(detailed.result); + expect(projected).not.toHaveProperty('disposition'); + expect(detailed.disposition).toBe('clean-absent'); + + fetchCalls = 0; + const firstDetailed = (agent as any).syncExactKnowledgeAssetsFromPeerDetailed( + PEER_A, + 'coalesced-cg', + [EXACT_UAL_7], + ); + const secondDetailed = (agent as any).syncExactKnowledgeAssetsFromPeerDetailed( + PEER_A, + 'coalesced-cg', + [EXACT_UAL_7], + ); + const [first, second] = await Promise.all([firstDetailed, secondDetailed]); + expect(fetchCalls).toBe(2); + expect(first.result).toBe(second.result); + expect(first.disposition).toBe('clean-absent'); + expect(second.disposition).toBe('clean-absent'); + + fetchCalls = 0; + const forwardOrder = (agent as any).syncExactKnowledgeAssetsFromPeer( + PEER_A, + 'coalesced-cg', + [EXACT_UAL_7, EXACT_UAL_8], + ); + const reverseOrder = (agent as any).syncExactKnowledgeAssetsFromPeerDetailed( + PEER_A, + 'coalesced-cg', + [EXACT_UAL_8, EXACT_UAL_7], + ); + const [forward, reverse] = await Promise.all([forwardOrder, reverseOrder]); + expect(fetchCalls).toBe(2); + expect(forward).toBe(reverse.result); + expect(reverse.disposition).toBe('clean-absent'); + } finally { + await agent.stop().catch(() => {}); + } + }); + it('serializes different-budget durable syncs for the same peer and Context Graph', async () => { const firstMetaFetch = deferred(); let fetchCalls = 0; diff --git a/packages/agent/test/sync-fresh-per-attempt.test.ts b/packages/agent/test/sync-fresh-per-attempt.test.ts index ef662783b9..bd0c2ebefc 100644 --- a/packages/agent/test/sync-fresh-per-attempt.test.ts +++ b/packages/agent/test/sync-fresh-per-attempt.test.ts @@ -1064,6 +1064,72 @@ describe('fetchSyncPages: fresh envelope + fresh messageId per retry attempt', ( expect(observedSessionIds.at(-1)).not.toBe('expired-responder-token'); }); + it('reports whether an offset-zero phase reused an unfinished responder snapshot', async () => { + const contextGraphId = 'offset-zero-session-evidence-cg'; + const checkpointKey = getSyncCheckpointKey( + REMOTE_PEER_ID, + contextGraphId, + false, + 'data', + ); + const checkpointStore = new MemorySyncCheckpointStore({ clock: () => Date.now() }); + checkpointStore.set(checkpointKey, 0); + checkpointStore.setResponderSession( + checkpointKey, + 'unfinished-offset-zero-token', + Date.now() + DURABLE_DATA_SYNC_SESSION_TTL_MS, + ); + const observedSessionIds: Array = []; + + const runFetch = (forceFreshSession = false) => runFetchWithFakeTimers(fetchSyncPages({ + ctx: makeCtx(), + remotePeerId: REMOTE_PEER_ID, + contextGraphId, + includeSharedMemory: false, + phase: 'data', + graphUri: GRAPH_URI, + deadline: Date.now() + 60_000, + syncPageTimeoutMs: 5_000, + syncRouterAttempts: 1, + syncPageRetryAttempts: 1, + syncPageSize: 1, + syncDeniedResponse: '#DENIED', + debugSyncProgress: false, + protocolSync: PROTOCOL_ID, + checkpointStore, + forceFreshSession, + buildSyncRequest: async ( + _contextGraphId, + _offset, + _limit, + _includeSharedMemory, + _remotePeerId, + _phase, + _snapshotRef, + _sinceBatchId, + syncSessionId, + ) => { + observedSessionIds.push(syncSessionId); + return new TextEncoder().encode('request'); + }, + parseAndFilter: singleQuadParser, + send: async () => new Uint8Array(), + logWarn: noopLog, + logInfo: noopLog, + logDebug: noopLog, + })); + + const reused = await runFetch(); + expect(reused.resumedFromOffset).toBe(0); + expect(reused.responderSessionStartedFresh).toBe(false); + expect(observedSessionIds.at(-1)).toBe('unfinished-offset-zero-token'); + + const fresh = await runFetch(true); + expect(fresh.resumedFromOffset).toBe(0); + expect(fresh.responderSessionStartedFresh).toBe(true); + expect(observedSessionIds.at(-1)).not.toBe('unfinished-offset-zero-token'); + }); + it('drops a RESUMED session that aborts with a GENERIC transport error (network-path R1 fix)', async () => { // 2026-07-07 sync storm. Over the wire the responder's "superseded" message // is destroyed by the router's stream.abort, so the requester sees a diff --git a/packages/agent/test/sync-requester-progress.test.ts b/packages/agent/test/sync-requester-progress.test.ts index be0248d4ac..4f6a53d244 100644 --- a/packages/agent/test/sync-requester-progress.test.ts +++ b/packages/agent/test/sync-requester-progress.test.ts @@ -1,14 +1,18 @@ import { describe, expect, it } from 'vitest'; -import type { OperationContext } from '@origintrail-official/dkg-core'; +import { SYSTEM_CONTEXT_GRAPHS, type OperationContext } from '@origintrail-official/dkg-core'; import type { Quad } from '@origintrail-official/dkg-storage'; import { runDurableSync, + runDurableSyncDetailed, type DurableSyncFetchRequest, type DurableSyncStoreInsertRequest, } from '../src/sync/requester/durable-sync.js'; import { uniformDurableSyncBudget } from './durable-sync-test-helpers.js'; import { runSharedMemorySync } from '../src/sync/requester/shared-memory-sync.js'; -import type { SyncPageResult } from '../src/sync/requester/page-fetch.js'; +import { + SyncPageAccumulationLimitError, + type SyncPageResult, +} from '../src/sync/requester/page-fetch.js'; import { markSyncTransportFailure } from '../src/sync/error-tags.js'; function recorder(impl: (...args: A) => R) { @@ -25,6 +29,7 @@ function durableFetchRecorder( const ctx = { kind: 'system', id: 'test', startedAt: 0 } as OperationContext; const noop = () => {}; +const EXACT_UAL = 'did:dkg:base:84532/0x1111111111111111111111111111111111111111/7'; function pageResult( contextGraphId: string, @@ -35,6 +40,7 @@ function pageResult( quads: [], bytesReceived: 0, resumedFromOffset: 0, + responderSessionStartedFresh: true, nextOffset: 0, checkpointKey: `${contextGraphId}:${phase}`, completed: true, @@ -1203,3 +1209,225 @@ describe('sync requester progress accounting', () => { expect(deleteCheckpoint.calls).toContainEqual(['large-swm:snapshot:snapshot-ref']); }); }); + +describe('exact durable fetch disposition', () => { + async function runExact(options: { + meta?: Partial; + data?: Partial; + rawMeta?: Quad[]; + rawData?: Quad[]; + rejectedKcs?: number; + dataRejectedMissingMeta?: number; + fetchError?: Error; + abortAfterMeta?: boolean; + } = {}) { + const controller = new AbortController(); + return runDurableSyncDetailed({ + ctx, + remotePeerId: 'exact-peer', + contextGraphIds: ['exact-cg'], + durableSyncBudget: uniformDurableSyncBudget(() => Date.now() + 60_000), + exactAssetUalsFor: () => [EXACT_UAL], + fetchSyncPages: async ({ phase }) => { + if (options.fetchError) throw options.fetchError; + const page = pageResult('exact-cg', phase, { + quads: phase === 'meta' ? (options.rawMeta ?? []) : (options.rawData ?? []), + ...(phase === 'meta' ? options.meta : options.data), + }); + if (phase === 'meta' && options.abortAfterMeta) { + controller.abort(new Error('cancelled after exact metadata')); + } + return page; + }, + signal: controller.signal, + processDurableBatchInWorker: async (data, meta) => ({ + ...durableProcessResult(), + verifiedData: data, + verifiedMeta: meta, + totalFetchedDataQuads: data.length, + totalFetchedMetaQuads: meta.length, + emptyResponses: data.length === 0 && meta.length === 0 ? 1 : 0, + rejectedKcs: options.rejectedKcs ?? 0, + dataRejectedMissingMeta: options.dataRejectedMissingMeta ?? 0, + }), + storeInsert: async () => {}, + deleteCheckpoint: () => {}, + setCheckpoint: () => {}, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + } + + it('distinguishes fresh clean absence without changing public completion', async () => { + const detailed = await runExact(); + const projected = await runDurableSync({ + ctx, + remotePeerId: 'exact-peer-public', + contextGraphIds: ['exact-cg'], + durableSyncBudget: uniformDurableSyncBudget(() => Date.now() + 60_000), + exactAssetUalsFor: () => [EXACT_UAL], + fetchSyncPages: async ({ phase }) => pageResult('exact-cg', phase), + processDurableBatchInWorker: async () => durableProcessResult(), + storeInsert: async () => {}, + deleteCheckpoint: () => {}, + setCheckpoint: () => {}, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(detailed.exactFetchDisposition).toBe('clean-absent'); + expect(detailed.result.complete).toBe(false); + expect(projected.complete).toBe(false); + expect(projected).not.toHaveProperty('exactFetchDisposition'); + }); + + it('classifies returned exact descriptor and content as found', async () => { + const assertionGraph = 'did:dkg:context-graph:exact-cg/_verifiable_memory/asset/7'; + const detailed = await runExact({ + rawMeta: [ + { + subject: EXACT_UAL, + predicate: 'http://dkg.io/ontology/kaUal', + object: EXACT_UAL, + graph: 'did:dkg:context-graph:exact-cg/_meta', + } as Quad, + { + subject: EXACT_UAL, + predicate: 'http://dkg.io/ontology/assertionGraph', + object: assertionGraph, + graph: 'did:dkg:context-graph:exact-cg/_meta', + } as Quad, + ], + rawData: [{ + subject: 'http://example.com/entity', + predicate: 'http://example.com/value', + object: '"present"', + graph: assertionGraph, + } as Quad], + meta: { nextOffset: 2 }, + data: { nextOffset: 1 }, + }); + + expect(detailed.exactFetchDisposition).toBe('found'); + }); + + it.each([ + ['resumed empty suffix', { meta: { resumedFromOffset: 4, nextOffset: 4 } }], + ['reused offset-zero responder session', { meta: { responderSessionStartedFresh: false } }], + ['resumed empty data suffix', { data: { resumedFromOffset: 4, nextOffset: 4 } }], + ['reused offset-zero data responder session', { data: { responderSessionStartedFresh: false } }], + ['partial phase', { data: { completed: false } }], + ['timed out phase', { data: { completed: false, timedOut: true } }], + ['integrity rejection', { rejectedKcs: 1 }], + ['missing metadata rejection', { dataRejectedMissingMeta: 1 }], + ['denial', { fetchError: deniedError() }], + ['abort', { abortAfterMeta: true }], + ])('keeps %s incomplete', async (_label, options) => { + const detailed = await runExact(options); + expect(detailed.exactFetchDisposition).toBe('incomplete'); + }); + + it('keeps an old responder filtered prefix incomplete', async () => { + const detailed = await runExact({ + rawMeta: [quad('did:dkg:other-asset')], + rawData: [quad('http://example.com/unrelated')], + meta: { nextOffset: 1 }, + data: { nextOffset: 1 }, + }); + expect(detailed.exactFetchDisposition).toBe('incomplete'); + }); + + it('does not verify or store an exact phase rejected by its accumulation limit', async () => { + const processDurableBatchInWorker = recorder(async () => durableProcessResult()); + const storeInsert = recorder(async (_request: DurableSyncStoreInsertRequest) => {}); + const fetchSyncPages = durableFetchRecorder(async () => { + throw new SyncPageAccumulationLimitError('bytes', 11, 10); + }); + const detailed = await runDurableSyncDetailed({ + ctx, + remotePeerId: 'legacy-exact-peer', + contextGraphIds: ['exact-cg'], + durableSyncBudget: uniformDurableSyncBudget(() => Date.now() + 60_000), + exactAssetUalsFor: () => [EXACT_UAL], + fetchSyncPages, + processDurableBatchInWorker, + storeInsert, + deleteCheckpoint: () => {}, + setCheckpoint: () => {}, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(detailed.exactFetchDisposition).toBe('incomplete'); + expect(detailed.result.failedPhases).toBe(1); + expect(fetchSyncPages.calls).toHaveLength(1); + expect(fetchSyncPages.calls[0][0].phase).toBe('meta'); + expect(processDurableBatchInWorker.calls).toHaveLength(0); + expect(storeInsert.calls).toHaveLength(0); + }); + + it('does not treat skipped agents metadata as a clean exact response', async () => { + const fetchedPhases: string[] = []; + const detailed = await runDurableSyncDetailed({ + ctx, + remotePeerId: 'exact-agents-peer', + contextGraphIds: [SYSTEM_CONTEXT_GRAPHS.AGENTS], + syncAgentsMeta: false, + durableSyncBudget: uniformDurableSyncBudget(() => Date.now() + 60_000), + exactAssetUalsFor: () => [EXACT_UAL], + fetchSyncPages: async ({ contextGraphId, phase }) => { + fetchedPhases.push(phase); + return pageResult(contextGraphId, phase); + }, + processDurableBatchInWorker: async () => durableProcessResult(), + storeInsert: async () => {}, + deleteCheckpoint: () => {}, + setCheckpoint: () => {}, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(fetchedPhases).toEqual(['data']); + expect(detailed.exactFetchDisposition).toBe('incomplete'); + }); + + it('aggregates exact outcomes across Context Graphs without overwriting an incomplete result', async () => { + const detailed = await runDurableSyncDetailed({ + ctx, + remotePeerId: 'exact-multi-cg-peer', + contextGraphIds: ['exact-incomplete-cg', 'exact-clean-cg'], + durableSyncBudget: uniformDurableSyncBudget(() => Date.now() + 60_000), + exactAssetUalsFor: () => [EXACT_UAL], + fetchSyncPages: async ({ contextGraphId, phase }) => pageResult(contextGraphId, phase, { + ...(contextGraphId === 'exact-incomplete-cg' && phase === 'data' + ? { completed: false } + : {}), + }), + processDurableBatchInWorker: async () => durableProcessResult(), + storeInsert: async () => {}, + deleteCheckpoint: () => {}, + setCheckpoint: () => {}, + logInfo: noop, + logWarn: noop, + logDebug: noop, + }); + + expect(detailed.exactFetchDisposition).toBe('incomplete'); + }); + + it('keeps normalization failures on the asynchronous requester boundary', async () => { + let publicResult: ReturnType | undefined; + let detailedResult: ReturnType | undefined; + + expect(() => { + publicResult = runDurableSync(null as never); + detailedResult = runDurableSyncDetailed(null as never); + }).not.toThrow(); + await expect(publicResult).rejects.toThrow(); + await expect(detailedResult).rejects.toThrow(); + }); +}); diff --git a/packages/agent/test/sync-single-use-policy-wiring.test.ts b/packages/agent/test/sync-single-use-policy-wiring.test.ts index dbdc4917f9..c4749a4fcd 100644 --- a/packages/agent/test/sync-single-use-policy-wiring.test.ts +++ b/packages/agent/test/sync-single-use-policy-wiring.test.ts @@ -26,6 +26,10 @@ vi.mock('../src/sync/requester/page-fetch.js', () => ({ import { PROTOCOL_SYNC } from '@origintrail-official/dkg-core'; import { LifecycleSyncMethods } from '../src/dkg-agent-lifecycle.js'; +import { + MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + MAX_EXACT_SYNC_PHASE_QUADS_PER_ASSET, +} from '../src/sync/exact-assets.js'; describe('LifecycleSyncMethods sync transport policy wiring', () => { beforeEach(() => { @@ -68,4 +72,42 @@ describe('LifecycleSyncMethods sync transport policy wiring', () => { }, ); }); + + it('wires exact-asset accumulation limits into the lifecycle page fetch', async () => { + const agent: any = { + node: { stopSignal: new AbortController().signal }, + messenger: { sendToPeer: vi.fn(async () => new Uint8Array()) }, + syncCheckpoints: new Map(), + buildSyncRequest: vi.fn(), + getOrCreateSyncVerifyWorker: () => ({ parseAndFilter: vi.fn() }), + log: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() }, + }; + const uals = [ + 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/7', + 'did:dkg:base:84532/0x0000000000000000000000000000000000000001/8', + ]; + + await (LifecycleSyncMethods.prototype.fetchSyncPages as any).call( + agent, + { operationId: 'exact-limit-wiring' }, + 'remote-peer', + 'public-context-graph', + false, + 'data', + 'did:dkg:public-context-graph', + Date.now() + 60_000, + undefined, + undefined, + undefined, + undefined, + undefined, + uals, + ); + + expect(fetchSyncPagesMock).toHaveBeenCalledWith(expect.objectContaining({ + assetUals: uals, + maxAcceptedBytes: 2 * MAX_EXACT_SYNC_PHASE_BYTES_PER_ASSET, + maxAcceptedQuads: 2 * MAX_EXACT_SYNC_PHASE_QUADS_PER_ASSET, + })); + }); }); diff --git a/packages/agent/test/vm-reconcile-self-prime.test.ts b/packages/agent/test/vm-reconcile-self-prime.test.ts index 50545acced..0896e657c6 100644 --- a/packages/agent/test/vm-reconcile-self-prime.test.ts +++ b/packages/agent/test/vm-reconcile-self-prime.test.ts @@ -10,7 +10,7 @@ * ontology OnChainId quad is locally present gets bound + persisted, and the * sweep then triggers its reconcile. Hermetic — MockChainAdapter, no network. */ -import { afterEach, describe, it, expect } from 'vitest'; +import { afterEach, describe, it, expect, vi } from 'vitest'; import { MockChainAdapter } from '@origintrail-official/dkg-chain'; import { DKG_ONTOLOGY, @@ -21,12 +21,23 @@ import { import type { TripleStore } from '@origintrail-official/dkg-storage'; import { DKGAgent } from '../src/index.js'; +function deferred(): { + promise: Promise; + resolve: (value: T) => void; +} { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { resolve = res; }); + return { promise, resolve }; +} + interface AgentInternals { runVmReconcileSweep(): Promise; selfPrimeSubscriptionOnChainId( localCgId: string, sub: { subscribed: boolean; coreHosted?: boolean; onChainId?: string }, targetOnChainId?: bigint, + isCurrent?: () => boolean, + signal?: AbortSignal, ): Promise; handleKARegisteredNudge(onChainId: string, kaId: bigint, ctx: unknown): Promise; subscribedContextGraphs: Map; @@ -134,6 +145,171 @@ describe('GH #1098 — VM reconcile sweep self-primes onChainId for a pre-subscr expect(internals.subscribedContextGraphs.get(CG_MATCH)?.onChainId).toBe(ON_MATCH); }); + it('does not bind or persist a replacement subscription after delayed self-prime is invalidated', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeLifecycleFence', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'gh1098-stale-self-prime'; + const original: { subscribed: boolean; onChainId?: string } = { subscribed: true }; + const replacement = { subscribed: true, onChainId: '9002' }; + internals.subscribedContextGraphs.set(localCgId, original); + + const lookup = deferred(); + let receivedSignal: AbortSignal | undefined; + (internals as any).getContextGraphOnChainId = async ( + _id: string, + options: { signal?: AbortSignal }, + ) => { + receivedSignal = options.signal; + return lookup.promise; + }; + const persist = vi.fn(); + (internals as any).persistContextGraphSubscription = persist; + let current = true; + const controller = new AbortController(); + + const prime = internals.selfPrimeSubscriptionOnChainId( + localCgId, + original, + undefined, + () => current, + controller.signal, + ); + await Promise.resolve(); + current = false; + controller.abort(); + internals.subscribedContextGraphs.set(localCgId, replacement); + lookup.resolve('9001'); + + await expect(prime).resolves.toBeNull(); + expect(receivedSignal).toBe(controller.signal); + expect(original.onChainId).toBeUndefined(); + expect(internals.subscribedContextGraphs.get(localCgId)).toBe(replacement); + expect(persist).not.toHaveBeenCalled(); + }); + + it('does not overwrite a same-object binding that lands during self-prime', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeSameObjectFence', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'gh1098-same-object-self-prime'; + const original: { subscribed: boolean; onChainId?: string } = { subscribed: true }; + internals.subscribedContextGraphs.set(localCgId, original); + + const lookup = deferred(); + (internals as any).getContextGraphOnChainId = async () => lookup.promise; + const persist = vi.fn(); + (internals as any).persistContextGraphSubscription = persist; + + const prime = internals.selfPrimeSubscriptionOnChainId(localCgId, original); + await Promise.resolve(); + original.onChainId = '9002'; + lookup.resolve('9001'); + + await expect(prime).resolves.toBeNull(); + expect(original.onChainId).toBe('9002'); + expect(persist).not.toHaveBeenCalled(); + }); + + it('strict-persists the resolved binding before exposing it to live reconcile state', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeStrictOrdering', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'gh1098-strict-ordering'; + const original: { subscribed: boolean; onChainId?: string } = { subscribed: true }; + internals.subscribedContextGraphs.set(localCgId, original); + (internals as any).getContextGraphOnChainId = async () => '9010'; + const persistStrict = vi.fn(async ( + _id: string, + candidate: { onChainId?: string }, + ) => { + expect(candidate.onChainId).toBe('9010'); + expect(original.onChainId).toBeUndefined(); + }); + (internals as any).persistContextGraphSubscriptionStrict = persistStrict; + + await expect(internals.selfPrimeSubscriptionOnChainId(localCgId, original)) + .resolves.toBe('9010'); + + expect(persistStrict).toHaveBeenCalledOnce(); + expect(original.onChainId).toBe('9010'); + }); + + it('leaves self-prime unbound when strict persistence fails', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeStrictFailure', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'gh1098-strict-failure'; + const original: { subscribed: boolean; onChainId?: string } = { subscribed: true }; + internals.subscribedContextGraphs.set(localCgId, original); + (internals as any).getContextGraphOnChainId = async () => '9011'; + (internals as any).persistContextGraphSubscriptionStrict = async () => { + throw new Error('subscription store unavailable'); + }; + + await expect(internals.selfPrimeSubscriptionOnChainId(localCgId, original)) + .resolves.toBeNull(); + expect(original.onChainId).toBeUndefined(); + }); + + it('rechecks the binding generation after strict self-prime persistence', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeStrictGeneration', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'gh1098-strict-generation'; + const original: { subscribed: boolean; onChainId?: string } = { subscribed: true }; + internals.subscribedContextGraphs.set(localCgId, original); + (internals as any).getContextGraphOnChainId = async () => '9012'; + const persisted = deferred(); + let markPersistStarted!: () => void; + const persistStarted = new Promise((resolve) => { markPersistStarted = resolve; }); + (internals as any).persistContextGraphSubscriptionStrict = async () => { + markPersistStarted(); + await persisted.promise; + }; + + const prime = internals.selfPrimeSubscriptionOnChainId(localCgId, original); + await persistStarted; + (internals as any).bindSubscriptionOnChainId(localCgId, original, '9999'); + original.onChainId = undefined; + persisted.resolve(); + + await expect(prime).resolves.toBeNull(); + expect(original.onChainId).toBeUndefined(); + }); + + it('settles promptly on lifecycle abort even when the lookup ignores its signal', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeAbortRace', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + const localCgId = 'gh1098-abort-race'; + const original = { subscribed: true }; + internals.subscribedContextGraphs.set(localCgId, original); + (internals as any).getContextGraphOnChainId = async () => new Promise(() => undefined); + const persist = vi.fn(); + (internals as any).persistContextGraphSubscription = persist; + const controller = new AbortController(); + + const prime = internals.selfPrimeSubscriptionOnChainId( + localCgId, + original, + undefined, + () => !controller.signal.aborted, + controller.signal, + ); + await Promise.resolve(); + controller.abort(); + + await expect(prime).resolves.toBeNull(); + expect(persist).not.toHaveBeenCalled(); + }); + it('live KACG nudge handler: with multiple subscribed-unbound CGs, binds + reconciles ONLY the one matching the event id', async () => { // Exercises the EXACT branch the live `onKARegisteredToContextGraph` poller // hook runs (extracted to `handleKARegisteredNudge`), not just the underlying diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 888385b522..88a5c7c1c0 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -218,6 +218,7 @@ import { drainCatchupJobs } from './catchup-telemetry.js'; import { beginGracefulShutdown, buildProducerQuiescentTeardownSteps, + closeDaemonBackingStoresAfterTeardown, runProducerQuiescentTeardown, } from './teardown.js'; import { @@ -3834,16 +3835,17 @@ export async function runDaemonInner( ); } - // Stop the managed Oxigraph child AFTER the agent has stopped - // issuing store queries, so an in-flight SPARQL request never - // races the killed server. No-op when not using oxigraph-server. - await managedOxigraph - ?.stop() - .catch((err: any) => - log(`Managed Oxigraph stop error: ${err?.message ?? String(err)}`), - ); - dashDb.close(); - log("Stopped."); + // Stop backing stores only after physical agent work retired. A typed + // retirement timeout is a dependency quarantine, not an ordinary + // best-effort cleanup failure: killing Oxigraph/SQLite underneath the + // still-running writer would defeat the agent's fail-stop boundary. + const backingStoresClosed = await closeDaemonBackingStoresAfterTeardown(teardown, { + retryAgentStop: () => agent.stop(), + stopManagedOxigraph: () => managedOxigraph?.stop() ?? Promise.resolve(), + closeDashboardDb: () => dashDb.close(), + log, + }); + if (backingStoresClosed) log("Stopped."); } finally { await cleanupStateFiles(); } diff --git a/packages/cli/src/daemon/teardown.ts b/packages/cli/src/daemon/teardown.ts index 82d67a69bc..722184365d 100644 --- a/packages/cli/src/daemon/teardown.ts +++ b/packages/cli/src/daemon/teardown.ts @@ -36,6 +36,10 @@ // that belong to them — numerator and denominator from different // lifecycles. +import { + CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_ERROR_CODE, + VM_RECONCILE_SHUTDOWN_TIMEOUT_ERROR_CODE, +} from '@origintrail-official/dkg-agent'; import { CATCHUP_SHUTDOWN_DRAIN_BUDGET_MS } from './catchup-telemetry.js'; /** @@ -153,6 +157,20 @@ export interface TeardownStepFailure { export interface TeardownOutcome { /** Empty on a fully clean teardown. Order matches execution order. */ failures: TeardownStepFailure[]; + /** Agent physical work is still alive; backing stores must remain open. */ + dependencyQuarantined: boolean; +} + +const DEPENDENCY_QUARANTINE_ERROR_CODES = new Set([ + VM_RECONCILE_SHUTDOWN_TIMEOUT_ERROR_CODE, + CONTEXT_GRAPH_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT_ERROR_CODE, +]); + +function isDependencyQuarantineError(error: unknown): boolean { + return typeof error === 'object' + && error !== null + && 'code' in error + && DEPENDENCY_QUARANTINE_ERROR_CODES.has(String((error as { code?: unknown }).code)); } /** @@ -213,18 +231,58 @@ export async function runProducerQuiescentTeardown( log: (message: string) => void = () => {}, ): Promise { const failures: TeardownStepFailure[] = []; + let dependencyQuarantined = false; for (const step of TEARDOWN_ORDER) { try { await steps[step](); } catch (error) { failures.push({ step, error }); + if (step === 'stopAgent' && isDependencyQuarantineError(error)) { + dependencyQuarantined = true; + } log( `[shutdown] teardown step "${step}" failed: ` + `${error instanceof Error ? error.message : String(error)} — continuing with the rest.`, ); } } - return { failures }; + return { failures, dependencyQuarantined }; +} + +export async function closeDaemonBackingStoresAfterTeardown( + outcome: TeardownOutcome, + deps: { + retryAgentStop: () => Promise; + stopManagedOxigraph: () => Promise; + closeDashboardDb: () => void; + log: (message: string) => void; + }, +): Promise { + if (outcome.dependencyQuarantined) { + deps.log( + '[shutdown] backing stores remain live because agent physical work did not retire; ' + + 'retrying retirement until it succeeds or the outer shutdown deadline forces exit', + ); + while (true) { + await new Promise((resolve) => setTimeout(resolve, 100)); + try { + await deps.retryAgentStop(); + break; + } catch (error) { + if (isDependencyQuarantineError(error)) continue; + deps.log( + `[shutdown] agent retirement retry completed with a non-quarantine error: ` + + `${error instanceof Error ? error.message : String(error)}; continuing backing-store teardown`, + ); + break; + } + } + } + await deps.stopManagedOxigraph().catch((error: unknown) => { + deps.log(`Managed Oxigraph stop error: ${error instanceof Error ? error.message : String(error)}`); + }); + deps.closeDashboardDb(); + return true; } /** diff --git a/packages/cli/test/daemon-catchup-telemetry-shutdown.test.ts b/packages/cli/test/daemon-catchup-telemetry-shutdown.test.ts index 9b5060c289..6ee7196014 100644 --- a/packages/cli/test/daemon-catchup-telemetry-shutdown.test.ts +++ b/packages/cli/test/daemon-catchup-telemetry-shutdown.test.ts @@ -77,8 +77,10 @@ const { const { beginGracefulShutdown, buildProducerQuiescentTeardownSteps, + closeDaemonBackingStoresAfterTeardown, runProducerQuiescentTeardown, } = await import('../src/daemon/teardown.js'); +const { raceShutdownWithTimeout } = await import('../src/daemon/shutdown.js'); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -954,6 +956,83 @@ describe('A24 — a failing step never strands the steps after it', () => { const { steps } = recordingSteps(); const outcome = await runProducerQuiescentTeardown(steps); expect(outcome.failures).toEqual([]); + expect(outcome.dependencyQuarantined).toBe(false); + }); + + it.each([ + 'VmReconcileShutdownTimeout', + 'CG_MEMBERSHIP_PERSIST_SHUTDOWN_TIMEOUT', + ])('quarantines backing stores when stopAgent fails with %s', async (code) => { + const { steps } = recordingSteps(); + steps.stopAgent = async () => { + throw Object.assign(new Error('physical work still active'), { code }); + }; + const outcome = await runProducerQuiescentTeardown(steps); + const stopManagedOxigraph = vi.fn(async () => undefined); + const closeDashboardDb = vi.fn(); + const logged: string[] = []; + const retryAgentStop = vi.fn(async () => undefined); + + expect(outcome.dependencyQuarantined).toBe(true); + await expect(closeDaemonBackingStoresAfterTeardown(outcome, { + retryAgentStop, + stopManagedOxigraph, + closeDashboardDb, + log: (message) => logged.push(message), + })).resolves.toBe(true); + expect(retryAgentStop).toHaveBeenCalledOnce(); + expect(stopManagedOxigraph).toHaveBeenCalledOnce(); + expect(closeDashboardDb).toHaveBeenCalledOnce(); + expect(logged.join(' ')).toContain('backing stores remain live'); + }); + + it('still closes backing stores after an ordinary agent-stop failure', async () => { + const { steps } = recordingSteps('stopAgent'); + const outcome = await runProducerQuiescentTeardown(steps); + const stopManagedOxigraph = vi.fn(async () => undefined); + const closeDashboardDb = vi.fn(); + + await expect(closeDaemonBackingStoresAfterTeardown(outcome, { + retryAgentStop: vi.fn(async () => undefined), + stopManagedOxigraph, + closeDashboardDb, + log: () => undefined, + })).resolves.toBe(true); + expect(stopManagedOxigraph).toHaveBeenCalledOnce(); + expect(closeDashboardDb).toHaveBeenCalledOnce(); + }); + + it('keeps a permanent retirement quarantine pending through the hard shutdown deadline', async () => { + const outcome = { + failures: [{ + step: 'stopAgent' as const, + error: Object.assign(new Error('still active'), { code: 'VmReconcileShutdownTimeout' }), + }], + dependencyQuarantined: true, + }; + let canRetire = false; + const stopManagedOxigraph = vi.fn(async () => undefined); + const closeDashboardDb = vi.fn(); + const cleanup = closeDaemonBackingStoresAfterTeardown(outcome, { + retryAgentStop: async () => { + if (!canRetire) { + throw Object.assign(new Error('still active'), { code: 'VmReconcileShutdownTimeout' }); + } + }, + stopManagedOxigraph, + closeDashboardDb, + log: () => undefined, + }).then(() => undefined); + + await expect(raceShutdownWithTimeout(cleanup, 250, () => undefined)) + .resolves.toEqual({ forced: true }); + expect(stopManagedOxigraph).not.toHaveBeenCalled(); + expect(closeDashboardDb).not.toHaveBeenCalled(); + + canRetire = true; + await cleanup; + expect(stopManagedOxigraph).toHaveBeenCalledOnce(); + expect(closeDashboardDb).toHaveBeenCalledOnce(); }); }); diff --git a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts index c470a14edf..9b5e70c670 100644 --- a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts +++ b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts @@ -294,6 +294,7 @@ describe('runDaemonInner StorageACK timing wiring', () => { start: vi.fn(async () => undefined), stop: vi.fn(async () => undefined), publishProfile: vi.fn(async () => undefined), + ensureProfilePublished: vi.fn(async () => undefined), publishRelayRegistry: vi.fn(async () => undefined), ensureContextGraphLocal: vi.fn(async () => undefined), getSubscribedContextGraphs: vi.fn(() => new Map()), diff --git a/packages/publisher/src/metadata.ts b/packages/publisher/src/metadata.ts index 6252a40e6e..bc3d225072 100644 --- a/packages/publisher/src/metadata.ts +++ b/packages/publisher/src/metadata.ts @@ -1329,9 +1329,11 @@ export async function withMaterializationLock( metaGraph: string, ual: string, fn: () => Promise, + options: { signal?: AbortSignal } = {}, ): Promise { const key = `${metaGraph}\u0000${ual}`; const prev = _materializationLocks.get(key); + let entered = false; // Build our work promise so subsequent callers can chain after us // BEFORE we start awaiting prev (otherwise two near-simultaneous // callers would both see `prev === undefined` and run in parallel). @@ -1339,17 +1341,49 @@ export async function withMaterializationLock( if (prev) { try { await prev; } catch { /* prev's caller already handled it */ } } + if (options.signal?.aborted) { + throw new DOMException('Materialization lock wait aborted', 'AbortError'); + } + entered = true; return fn(); })(); _materializationLocks.set(key, work); - try { - return await work; - } finally { + // Cleanup follows the serialized tail, not the caller-facing abort race. If + // a waiter aborts behind an active owner, deleting the key immediately would + // let a third writer bypass that owner and violate the TOCTOU guarantee. + void work.finally(() => { // GC: if no one else queued after us, drop the entry so the map // doesn't grow unbounded across long-running daemons. if (_materializationLocks.get(key) === work) { _materializationLocks.delete(key); } + }).catch(() => undefined); + if (!options.signal) return work; + + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => { + // Once the critical section has started it is an atomic durability unit: + // its caller must observe its real completion before shutdown can close + // the store. Cancellation only removes callers still waiting for the + // previous owner; it must never detach an entered writer. + if (entered) return; + // An aborted waiter that never entered the critical section contributes + // no serialization work. Restore the prior owner as the visible tail so + // repeated stop/restart cycles cannot accumulate an unbounded promise + // chain behind one physically hung store mutation. + if (!entered && prev && _materializationLocks.get(key) === work) { + _materializationLocks.set(key, prev); + } + reject(new DOMException('Materialization lock wait aborted', 'AbortError')); + }; + if (options.signal!.aborted) onAbort(); + else options.signal!.addEventListener('abort', onAbort, { once: true }); + }); + try { + return entered ? await work : await Promise.race([work, aborted]); + } finally { + if (onAbort) options.signal.removeEventListener('abort', onAbort); } } diff --git a/packages/publisher/test/materialization-lock.test.ts b/packages/publisher/test/materialization-lock.test.ts index 39515d79b9..2db28a32c0 100644 --- a/packages/publisher/test/materialization-lock.test.ts +++ b/packages/publisher/test/materialization-lock.test.ts @@ -103,6 +103,55 @@ describe('withMaterializationLock — serialises check + write per (metaGraph, u }); expect(secondRan).toBe(true); }); + + it('lets an aborted waiter leave without bypassing the active same-key writer', async () => { + let releaseOwner!: () => void; + const ownerGate = new Promise((resolve) => { releaseOwner = resolve; }); + const owner = withMaterializationLock(LABEL_META, UAL, async () => ownerGate); + const controller = new AbortController(); + let waiterRan = false; + const waiter = withMaterializationLock(LABEL_META, UAL, async () => { + waiterRan = true; + }, { signal: controller.signal }); + + await Promise.resolve(); + controller.abort(); + await expect(waiter).rejects.toMatchObject({ name: 'AbortError' }); + expect(waiterRan).toBe(false); + + let successorRan = false; + const successor = withMaterializationLock(LABEL_META, UAL, async () => { + successorRan = true; + }); + await Promise.resolve(); + expect(successorRan).toBe(false); + + releaseOwner(); + await Promise.all([owner, successor]); + expect(successorRan).toBe(true); + }); + + it('waits for an entered atomic writer even when its signal aborts', async () => { + const controller = new AbortController(); + let release!: () => void; + let entered!: () => void; + const enteredGate = new Promise((resolve) => { entered = resolve; }); + const workGate = new Promise((resolve) => { release = resolve; }); + let settled = false; + const owner = withMaterializationLock(LABEL_META, UAL, async () => { + entered(); + await workGate; + return 'committed'; + }, { signal: controller.signal }).finally(() => { settled = true; }); + + await enteredGate; + controller.abort(); + await Promise.resolve(); + expect(settled).toBe(false); + + release(); + await expect(owner).resolves.toBe('committed'); + }); }); /**