diff --git a/packages/agent/src/context-graph-subscription-policy.ts b/packages/agent/src/context-graph-subscription-policy.ts new file mode 100644 index 0000000000..25681b0e31 --- /dev/null +++ b/packages/agent/src/context-graph-subscription-policy.ts @@ -0,0 +1,79 @@ +import type { + ContextGraphSub, + ContextGraphSubInput, + ContextGraphSubscriptionRecord, + ContextGraphSyncMode, +} from './dkg-agent-types.js'; + +export function normalizeLegacyContextGraphSubscriptionInput( + previous: ContextGraphSub | undefined, + next: ContextGraphSubInput, +): ContextGraphSub { + return { + ...next, + syncMode: next.syncMode ?? previous?.syncMode ?? 'always-on', + }; +} + +export function resolveContextGraphSyncMode(input: { + existing?: Pick; + requested?: ContextGraphSyncMode; + hasDormantDurableIntent: boolean; +}): ContextGraphSyncMode { + if ( + input.hasDormantDurableIntent + || (input.existing?.subscribed === true && input.existing.syncMode === 'always-on') + ) { + return 'always-on'; + } + return input.requested ?? input.existing?.syncMode ?? 'always-on'; +} + +export type ContextGraphSubscriptionPersistenceProjection = + | { action: 'skip'; persistMemberIntent: false } + | { action: 'delete'; persistMemberIntent: true } + | { + action: 'save'; + persistMemberIntent: boolean; + record: ContextGraphSubscriptionRecord; + }; + +/** + * Canonical durable projection for live Context Graph subscription state. + * + * On-demand member intent remains process-local. A Core hosting obligation is + * independently durable and therefore projects to a host-only row. Always-on + * member intent projects the complete live readiness state. + */ +export function projectContextGraphSubscriptionPersistence(input: { + contextGraphId: string; + subscription: ContextGraphSub | undefined; + syncScoped: boolean; +}): ContextGraphSubscriptionPersistenceProjection { + const sub = input.subscription; + if (sub?.syncMode === 'on-demand' && sub.coreHosted !== true) { + return { action: 'skip', persistMemberIntent: false }; + } + if (!sub?.subscribed && !sub?.coreHosted) { + return { action: 'delete', persistMemberIntent: true }; + } + + const persistMemberIntent = sub.syncMode !== 'on-demand'; + return { + action: 'save', + persistMemberIntent, + record: { + id: input.contextGraphId, + name: sub.name, + subscribed: persistMemberIntent && sub.subscribed, + synced: persistMemberIntent && sub.synced, + sharedMemorySynced: persistMemberIntent ? sub.sharedMemorySynced : false, + metaSynced: persistMemberIntent ? sub.metaSynced : false, + onChainId: sub.onChainId, + onChainHash: sub.onChainHash, + lastReconciledOrdinal: sub.lastReconciledOrdinal, + coreHosted: sub.coreHosted, + syncScoped: persistMemberIntent && input.syncScoped, + }, + }; +} diff --git a/packages/agent/src/dkg-agent-cg-registry.ts b/packages/agent/src/dkg-agent-cg-registry.ts index 115602f5b7..cd6a2c2619 100644 --- a/packages/agent/src/dkg-agent-cg-registry.ts +++ b/packages/agent/src/dkg-agent-cg-registry.ts @@ -1048,9 +1048,10 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { // one curator triple per node and `getContextGraphOwner`'s // `LIMIT 1` made ownership nondeterministic — any subscriber could // win the unordered query and look like the curator. - this.subscribeToContextGraph(opts.id); + this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); this.setContextGraphSubscription(opts.id, { name: opts.name, + syncMode: 'always-on', subscribed: true, synced: true, metaSynced: true, @@ -1108,9 +1109,10 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { this.contextGraphMetaProjection.markDirtyFromQuads(quads); await gm.ensureNewContextGraph(opts.id); - this.subscribeToContextGraph(opts.id); + this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); this.setContextGraphSubscription(opts.id, { name: opts.name, + syncMode: 'always-on', subscribed: true, synced: true, metaSynced: true, diff --git a/packages/agent/src/dkg-agent-context-graph.ts b/packages/agent/src/dkg-agent-context-graph.ts index 4317e4ba1b..d41a804774 100644 --- a/packages/agent/src/dkg-agent-context-graph.ts +++ b/packages/agent/src/dkg-agent-context-graph.ts @@ -696,6 +696,7 @@ export class ContextGraphMethods extends DKGAgentBase { this.setContextGraphSubscription(opts.id, { name: opts.name, + syncMode: 'always-on', subscribed: !opts.private, synced: true, metaSynced: true, @@ -817,7 +818,7 @@ export class ContextGraphMethods extends DKGAgentBase { } if (!opts.private) { - this.subscribeToContextGraph(opts.id); + this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); // Curated CGs: definition lives in _meta, NOT in ONTOLOGY. Do not // broadcast to the network — only invited nodes will discover it via @@ -1483,8 +1484,11 @@ export class ContextGraphMethods extends DKGAgentBase { const next = { ...sub, onChainHash: nameHash }; this.bindSubscriptionOnChainId(id, next, onChainId); this.setContextGraphSubscription(id, next, { persist: false }); + this.subscribeToContextGraph(id, { + trackSyncScope: true, + syncMode: 'always-on', + }); if (!next.subscribed) { - this.subscribeToContextGraph(id, { trackSyncScope: true }); this.log.info(ctx, `Subscribed to newly registered context graph "${id}"`); } this.persistContextGraphSubscription(id); diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 51f29f893c..01e6d967ee 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -474,6 +474,7 @@ import { type SyncReconcilerProbe, type SyncReconcilerBackoff, } from './dkg-agent-types.js'; +import { projectContextGraphSubscriptionPersistence } from './context-graph-subscription-policy.js'; import { authoritativeSyncPeerId, resolveCuratorSyncPeer, @@ -2338,6 +2339,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // upsert a minimal stub first. if (!this.subscribedContextGraphs.has(localId)) { this.setContextGraphSubscription(localId, { + syncMode: 'always-on', subscribed: false, synced: false, onChainHash: hashLower, @@ -2674,6 +2676,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { } const approvedSubscription: ContextGraphSub = { ...this.subscribedContextGraphs.get(contextGraphId), + syncMode: 'always-on', subscribed: true, pendingMeta: true, metaSynced: false, @@ -2742,6 +2745,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // update/finalization) wire up immediately as before. this.subscribeToContextGraph(contextGraphId, { deferSharedMemoryGossipSubscribe: true, + syncMode: 'always-on', // The exact approval snapshot was committed above. Scheduling // the ordinary background persistence here would reintroduce // untracked writes around the compensating transaction. @@ -3031,7 +3035,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // Subscribe to both system context graph GossipSub topics for (const systemContextGraph of [SYSTEM_CONTEXT_GRAPHS.AGENTS, SYSTEM_CONTEXT_GRAPHS.ONTOLOGY]) { - this.subscribeToContextGraph(systemContextGraph); + this.subscribeToContextGraph(systemContextGraph, { syncMode: 'always-on' }); } // Connect to bootstrap peers @@ -6512,9 +6516,10 @@ export class LifecycleSyncMethods extends DKGAgentBase { ? this.contextGraphWireId(next.onChainHash) : undefined; const nextWireId = nextOnChainHash ?? localWireId; - const canonicalNext = next.onChainHash === nextOnChainHash - ? next - : { ...next, onChainHash: nextOnChainHash }; + const canonicalNext: ContextGraphSub = { + ...next, + ...(next.onChainHash === nextOnChainHash ? {} : { onChainHash: nextOnChainHash }), + }; if ( previousWireId !== nextWireId && this.wireIdToLocalCgId.get(previousWireId) === contextGraphId @@ -6526,7 +6531,16 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (!canonicalNext.subscribed && !canonicalNext.coreHosted) { this.clearVmReconcileStateForContextGraph(contextGraphId); } - if (options?.persist !== false) { + // On-demand member subscriptions deliberately keep their live state and + // readiness process-local. A Core's independent hosting obligation is + // still durable, though: persistContextGraphSubscription writes a + // host-only snapshot without converting the member intent to always-on. + const persistence = projectContextGraphSubscriptionPersistence({ + contextGraphId, + subscription: canonicalNext, + syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), + }); + if (options?.persist !== false && persistence.action !== 'skip') { if (this.config.contextGraphSubscriptionStore) { const revision = this.nextContextGraphSubscriptionPersistRevision(contextGraphId); this.persistContextGraphSubscription( @@ -6537,9 +6551,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { }, ); } - if (canonicalNext.subscribed) { + if (persistence.persistMemberIntent && canonicalNext.subscribed) { this.persistLocalNodeMembership(contextGraphId); - } else { + } else if (persistence.persistMemberIntent) { this.deleteContextGraphMember(contextGraphId, 'node', this.peerId); } } @@ -6627,7 +6641,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { updateContextGraphSubscriptionRehydrationStatusAfterPersist(this: DKGAgent, contextGraphId: string, - next?: ContextGraphSub, + next?: Pick, ): void { const status = this.contextGraphSubscriptionRehydrationStatus; if (!status) return; @@ -6762,12 +6776,24 @@ export class LifecycleSyncMethods extends DKGAgentBase { return; } const sub = this.subscribedContextGraphs.get(contextGraphId); + const persistence = projectContextGraphSubscriptionPersistence({ + contextGraphId, + subscription: sub, + syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), + }); + if (persistence.action === 'skip') { + // Some lifecycle paths persist reconciliation watermarks directly + // instead of going through setContextGraphSubscription. Preserve the + // process-local lifetime at this lowest shared write boundary too. + this.clearContextGraphSubscriptionPersistRevisionStateIfIdle(contextGraphId); + return; + } // Persist member subscriptions AND (Phase D) public CGs this Core hosts — // the host-only record MUST survive restart so a Core that was offline // during a publish remembers it hosts the CG and fills its gap. Drop the // row only when the node neither subscribes to nor hosts the CG. this.beginContextGraphSubscriptionPersistRevision(contextGraphId, options?.revision); - if (!sub?.subscribed && !sub?.coreHosted) { + if (persistence.action === 'delete') { void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.delete(contextGraphId)) .then(() => { if ( @@ -6788,26 +6814,14 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); return; } - const record = { - id: contextGraphId, - name: sub.name, - subscribed: sub.subscribed, - synced: sub.synced, - sharedMemorySynced: sub.sharedMemorySynced, - metaSynced: sub.metaSynced, - onChainId: sub.onChainId, - onChainHash: sub.onChainHash, - lastReconciledOrdinal: sub.lastReconciledOrdinal, - coreHosted: sub.coreHosted, - syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), - }; + const record = persistence.record; void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.save(record)) .then(() => { if ( options?.updateRehydrationStatus === true && this.claimContextGraphSubscriptionPersistRevision(contextGraphId, options.revision) ) { - this.updateContextGraphSubscriptionRehydrationStatusAfterPersist(contextGraphId, sub); + this.updateContextGraphSubscriptionRehydrationStatusAfterPersist(contextGraphId, record); } }).catch((err) => { this.log.warn( @@ -6837,19 +6851,17 @@ export class LifecycleSyncMethods extends DKGAgentBase { `Cannot acknowledge join approval for "${contextGraphId}": active subscription state is missing`, ); } - const record = { - id: contextGraphId, - name: sub.name, - subscribed: sub.subscribed, - synced: sub.synced, - sharedMemorySynced: sub.sharedMemorySynced, - metaSynced: sub.metaSynced, - onChainId: sub.onChainId, - onChainHash: sub.onChainHash, - lastReconciledOrdinal: sub.lastReconciledOrdinal, - coreHosted: sub.coreHosted, + const persistence = projectContextGraphSubscriptionPersistence({ + contextGraphId, + subscription: sub, syncScoped: syncScoped ?? (this.config.syncContextGraphs ?? []).includes(contextGraphId), - }; + }); + if (persistence.action !== 'save' || !persistence.persistMemberIntent) { + throw new Error( + `Cannot acknowledge join approval for "${contextGraphId}": durable subscription intent is missing`, + ); + } + const record = persistence.record; // Queue behind any fire-and-forget writes scheduled by subscribe/mark so // this final authoritative snapshot is the last write before the ACK. await this.enqueueContextGraphSubscriptionPersistWrite( @@ -7258,6 +7270,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { const restorePendingMeta = hasJoinApproval && !approvedAgentAuthorized; this.setContextGraphSubscription(row.id, { name: row.name, + // Every row in the durable store predates or represents explicit + // restart persistence, so absence of a mode is always-on. + syncMode: 'always-on', subscribed: row.subscribed, synced: restorePendingMeta ? false : row.synced, sharedMemorySynced: restorePendingMeta ? false : row.sharedMemorySynced, @@ -7274,7 +7289,11 @@ export class LifecycleSyncMethods extends DKGAgentBase { this.trackSyncContextGraph(row.id); } if (row.subscribed) { - this.subscribeToContextGraph(row.id, { trackSyncScope: false, persist: false }); + this.subscribeToContextGraph(row.id, { + trackSyncScope: false, + persist: false, + syncMode: 'always-on', + }); this.persistLocalNodeMembership(row.id, 'rehydrated-subscription'); } // Upgrade/self-heal path for late private-CG members whose payload and diff --git a/packages/agent/src/dkg-agent-publish.ts b/packages/agent/src/dkg-agent-publish.ts index f0a5feac5a..7967c1a23a 100644 --- a/packages/agent/src/dkg-agent-publish.ts +++ b/packages/agent/src/dkg-agent-publish.ts @@ -2495,9 +2495,9 @@ export class PublishMethods extends DKGAgentBase { this.contextGraphMetaProjection.markDirtyFromQuads(quads); await gm.ensureContextGraph(contextGraphId); await this.store.flush?.(); - this.subscribeToContextGraph(contextGraphId); + const promotedSub = this.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); this.setContextGraphSubscription(contextGraphId, { - ...existingSub, + ...promotedSub, name, subscribed: true, synced: true, diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index bfbcac3441..854797a226 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -1128,6 +1128,7 @@ export class SwmHostModeMethods extends DKGAgentBase { // translate either direction without an extra RPC. if (storageCgId !== contextGraphId) { const storageSubscription = this.subscribedContextGraphs.get(storageCgId) ?? { + syncMode: 'always-on' as const, subscribed: false, synced: false, pendingMeta: true, @@ -1696,6 +1697,7 @@ export class SwmHostModeMethods extends DKGAgentBase { // that did not create or join the CG. if (!this.subscribedContextGraphs.has(wireId)) { this.setContextGraphSubscription(wireId, { + syncMode: 'always-on', subscribed: false, synced: false, onChainHash: wireId, @@ -2448,7 +2450,13 @@ export class SwmHostModeMethods extends DKGAgentBase { existing.coreHosted = true; next = existing; } else { - next = { subscribed: false, synced: false, onChainId: numericStr, coreHosted: true }; + next = { + syncMode: 'always-on', + subscribed: false, + synced: false, + onChainId: numericStr, + coreHosted: true, + }; } this.setContextGraphSubscription(localCgId, next); this.log.info( diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 4893d23842..c13b8f8d03 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -349,6 +349,7 @@ import { type DKGAgentConfig, type ReplicationEvent, } from './dkg-agent-types.js'; +import { resolveContextGraphSyncMode } from './context-graph-subscription-policy.js'; import { normalizePublishContextGraphId, isPublishAsyncQuadEnvelope, @@ -384,11 +385,29 @@ import { DKGAgentBase } from './dkg-agent-base.js'; import type { DKGAgent } from './dkg-agent.js'; export class SwmSubstrateMethods extends DKGAgentBase { - subscribeToContextGraph(this: DKGAgent, contextGraphId: string, options?: { trackSyncScope?: boolean; persist?: boolean; deferSharedMemoryGossipSubscribe?: boolean }): void { + subscribeToContextGraph(this: DKGAgent, contextGraphId: string, options?: { + trackSyncScope?: boolean; + persist?: boolean; + deferSharedMemoryGossipSubscribe?: boolean; + syncMode?: 'on-demand' | 'always-on'; + }): ContextGraphSub { if (options?.trackSyncScope !== false) { this.trackSyncContextGraph(contextGraphId); } + const existing = this.subscribedContextGraphs.get(contextGraphId); + // Opening an already durable graph must never silently downgrade it to a + // process-local subscription. An explicit always-on request may promote an + // existing on-demand subscription, while an omitted mode preserves the + // current lifetime (or the legacy always-on default for a new graph). + const syncMode = resolveContextGraphSyncMode({ + existing, + requested: options?.syncMode, + hasDormantDurableIntent: + this.contextGraphSubscriptionRehydrationStatus?.dormantIds.includes(contextGraphId) === true, + }); + const persist = syncMode === 'on-demand' ? false : options?.persist; + // SWM gossip subscribe runs `canReadContextGraph` against the local // `_meta` graph. On a fresh `join-approved` notification the curator // has just written the allowlist into ITS _meta, but the requesting @@ -407,15 +426,19 @@ export class SwmSubstrateMethods extends DKGAgentBase { if (!deferSwmGossip) { this.queueSharedMemoryGossipSubscription(contextGraphId); } - const existing = this.subscribedContextGraphs.get(contextGraphId); - if (!existing?.subscribed) { - this.setContextGraphSubscription( + if (!existing?.subscribed || existing.syncMode !== syncMode) { + return this.setContextGraphSubscription( contextGraphId, - { ...existing, subscribed: true, synced: existing?.synced ?? false }, - { persist: options?.persist }, + { + ...existing, + subscribed: true, + synced: existing?.synced ?? false, + syncMode, + }, + { persist }, ); } - return; + return existing; } this.gossipRegistered.add(contextGraphId); @@ -425,11 +448,15 @@ export class SwmSubstrateMethods extends DKGAgentBase { this.gossip.subscribe(publishTopic); this.gossip.subscribe(appTopic); - const existing = this.subscribedContextGraphs.get(contextGraphId); - this.setContextGraphSubscription( + const subscription = this.setContextGraphSubscription( contextGraphId, - { ...existing, subscribed: true, synced: existing?.synced ?? false }, - { persist: options?.persist }, + { + ...existing, + subscribed: true, + synced: existing?.synced ?? false, + syncMode, + }, + { persist }, ); this.gossip.onMessage(publishTopic, async (_topic, data, from) => { @@ -454,6 +481,8 @@ export class SwmSubstrateMethods extends DKGAgentBase { const fh = this.getOrCreateFinalizationHandler(); await fh.handleFinalizationMessage(data, contextGraphId, from); }); + + return subscription; } /** diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index e99152f1d6..d6accdf49c 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -605,9 +605,21 @@ export interface ChatSendResult { // ── Context-graph surface ─────────────────────────────────────────── +/** + * Lifetime of an edge node's active Context Graph synchronization intent. + * + * `on-demand` remains active only for the current process. `always-on` is + * restart-durable through the configured subscription store. Persisted rows + * written before this distinction existed are therefore implicitly + * `always-on` for backward compatibility. + */ +export type ContextGraphSyncMode = 'on-demand' | 'always-on'; + /** Tracks the subscription and sync state of a context graph. */ export interface ContextGraphSub { name?: string; + /** Requested synchronization lifetime, normalized before entering live state. */ + syncMode: ContextGraphSyncMode; /** GossipSub topics are active for this context graph. */ subscribed: boolean; /** Definition triples exist in the local triple store. */ @@ -685,6 +697,17 @@ export interface ContextGraphSub { pendingMeta?: boolean; } +/** + * Legacy compatibility input for the standalone gossip handler only. + * + * Agent-owned live-state mutations use normalized {@link ContextGraphSub} + * values and must choose a synchronization lifetime explicitly. This shape is + * retained solely for older standalone handler callbacks that predate modes. + */ +export type ContextGraphSubInput = Omit & { + syncMode?: ContextGraphSyncMode; +}; + /** * Metadata that passive discovery is allowed to contribute to the local * Context Graph catalogue. diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 85f8ff7c3f..dceb4c89ed 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -337,6 +337,7 @@ import { type PeerDiagnostics, type ChatSendResult, type ContextGraphSub, + type ContextGraphSyncMode, type ContextGraphDiscoveryMetadata, type ContextGraphDiscoveryOptions, type ContextGraphSubscriptionRecord, @@ -449,6 +450,7 @@ export type { PeerDiagnostics, ChatSendResult, ContextGraphSub, + ContextGraphSyncMode, ContextGraphDiscoveryMetadata, ContextGraphDiscoveryOptions, ContextGraphSubscriptionRecord, @@ -1226,6 +1228,7 @@ export class DKGAgent extends DKGAgentBase { const existing = this.subscribedContextGraphs.get(contextGraphId); const next: ContextGraphSub = { ...existing, + syncMode: existing?.syncMode ?? 'always-on', name: metadata.name ?? existing?.name, subscribed: existing?.subscribed === true, synced: existing?.synced === true, @@ -1252,6 +1255,7 @@ export class DKGAgent extends DKGAgentBase { if (!existing && (this.config.nodeRole ?? 'edge') === 'core') { this.subscribeToContextGraph(contextGraphId, { trackSyncScope: options.trackSyncScope, + syncMode: 'always-on', }); } diff --git a/packages/agent/src/gossip-publish-handler.ts b/packages/agent/src/gossip-publish-handler.ts index 69db636e4e..9a7c5c9589 100644 --- a/packages/agent/src/gossip-publish-handler.ts +++ b/packages/agent/src/gossip-publish-handler.ts @@ -31,7 +31,12 @@ import { } from '@origintrail-official/dkg-publisher'; import { ethers } from 'ethers'; import type { ContextGraphMetaRecord } from './context-graph-meta-projection.js'; -import type { ContextGraphDiscoveryMetadata, ContextGraphSub } from './dkg-agent-types.js'; +import type { + ContextGraphDiscoveryMetadata, + ContextGraphSub, + ContextGraphSubInput, +} from './dkg-agent-types.js'; +import { normalizeLegacyContextGraphSubscriptionInput } from './context-graph-subscription-policy.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; export type GossipPhaseCallback = (phase: string, status: 'start' | 'end') => void; @@ -195,15 +200,17 @@ export class GossipPublishHandler { private setContextGraphSubscription( id: string, - next: ContextGraphSub, + next: ContextGraphSubInput, options?: { persist?: boolean }, ): void { + const previous = this.subscribedContextGraphs.get(id); + const normalized = normalizeLegacyContextGraphSubscriptionInput(previous, next); const setter = this.callbacks.setContextGraphSubscription; if (setter) { - setter(id, next, options); + setter(id, normalized, options); return; } - this.subscribedContextGraphs.set(id, next); + this.subscribedContextGraphs.set(id, normalized); } private recordDiscoveredContextGraph(id: string, metadata: ContextGraphDiscoveryMetadata): void { diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index f9724c8f74..7c8e6cfe62 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -197,6 +197,7 @@ export { type Rfc64CatalogAccessPolicyAuthorityConfigV1, type DKGAgentACKTransportOptions, type ContextGraphSub, + type ContextGraphSyncMode, type ContextGraphDiscoveryMetadata, type ContextGraphDiscoveryOptions, type PublishOpts, diff --git a/packages/agent/test/agent.part-15.test.ts b/packages/agent/test/agent.part-15.test.ts index 500c28ca79..e5b3f23693 100644 --- a/packages/agent/test/agent.part-15.test.ts +++ b/packages/agent/test/agent.part-15.test.ts @@ -3,6 +3,46 @@ import { describe, it, expect, beforeAll, afterAll, vi, DKGAgentWallet, buildAge let _fileSnapshot: string; + +function createContextGraphPersistenceFixture() { + const persisted = new Map(); + const persistedMembers = new Map(); + const subscriptionStore = { + loadAll: async () => [...persisted.values()], + save: async (record: any) => { + persisted.set(record.id, { ...record }); + }, + delete: async (contextGraphId: string) => { + persisted.delete(contextGraphId); + }, + }; + const membershipStore = { + upsert: async (record: any) => { + persistedMembers.set( + `${record.contextGraphId}|${record.principalType}|${record.principalId}`, + { ...record }, + ); + }, + delete: async (contextGraphId: string, principalType: string, principalId: string) => { + persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); + }, + }; + return { persisted, persistedMembers, subscriptionStore, membershipStore }; +} + +async function createAgentWithContextGraphPersistence( + name: string, + fixture: ReturnType, +) { + return DKGAgent.create({ + name, + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: fixture.subscriptionStore, + contextGraphMembershipStore: fixture.membershipStore, + }); +} + beforeAll(async () => { _fileSnapshot = await takeSnapshot(); const { hubAddress } = getSharedContext(); @@ -68,33 +108,9 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => it('persists runtime subscriptions and rehydrates them on restart', async () => { - const persisted = new Map(); - const persistedMembers = new Map(); - const subscriptionStore = { - loadAll: async () => [...persisted.values()], - save: async (record: any) => { - persisted.set(record.id, { ...record }); - }, - delete: async (contextGraphId: string) => { - persisted.delete(contextGraphId); - }, - }; - const membershipStore = { - upsert: async (record: any) => { - persistedMembers.set(`${record.contextGraphId}|${record.principalType}|${record.principalId}`, { ...record }); - }, - delete: async (contextGraphId: string, principalType: string, principalId: string) => { - persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); - }, - }; - - const agentA = await DKGAgent.create({ - name: 'PersistedSubscriptionsA', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const agentA = await createAgentWithContextGraphPersistence('PersistedSubscriptionsA', fixture); let agentAPeerId = ''; try { @@ -130,18 +146,13 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => source: 'subscription', }); - const agentB = await DKGAgent.create({ - name: 'PersistedSubscriptionsB', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const agentB = await createAgentWithContextGraphPersistence('PersistedSubscriptionsB', fixture); try { await agentB.start(); expect(agentB.getSubscribedContextGraphs().get('persisted-cg')).toMatchObject({ subscribed: true, + syncMode: 'always-on', synced: true, sharedMemorySynced: true, metaSynced: true, @@ -162,6 +173,278 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => }); + it('keeps on-demand subscriptions process-local until explicitly promoted', async () => { + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const agentA = await createAgentWithContextGraphPersistence('OnDemandSubscriptionLifetimeA', fixture); + + try { + await agentA.start(); + agentA.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + agentA.markContextGraphSubscriptionState('selected-cg', { + synced: true, + sharedMemorySynced: true, + metaSynced: true, + }); + (agentA as any).persistContextGraphSubscription('selected-cg'); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(agentA.getSubscribedContextGraphs().get('selected-cg')).toMatchObject({ + subscribed: true, + syncMode: 'on-demand', + synced: true, + }); + expect(persisted.has('selected-cg')).toBe(false); + expect(persistedMembers.has(`selected-cg|node|${agentA.peerId}`)).toBe(false); + } finally { + await agentA.stop().catch(() => {}); + } + + const agentB = await createAgentWithContextGraphPersistence('OnDemandSubscriptionLifetimeB', fixture); + try { + await agentB.start(); + expect(agentB.getSubscribedContextGraphs().get('selected-cg')).toBeUndefined(); + + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(persistedMembers.has(`selected-cg|node|${agentB.peerId}`)).toBe(false); + + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'always-on' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(persisted.get('selected-cg')).toMatchObject({ + id: 'selected-cg', + subscribed: true, + synced: false, + }); + expect(agentB.getSubscribedContextGraphs().get('selected-cg')?.syncMode).toBe('always-on'); + expect(persistedMembers.get(`selected-cg|node|${agentB.peerId}`)).toMatchObject({ + contextGraphId: 'selected-cg', + principalType: 'node', + principalId: agentB.peerId, + status: 'active', + source: 'subscription', + }); + + // A later UI open is on-demand, but must not silently downgrade an + // operator's explicit always-on choice. + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + expect(agentB.getSubscribedContextGraphs().get('selected-cg')?.syncMode).toBe('always-on'); + } finally { + await agentB.stop().catch(() => {}); + } + }); + + + it('preserves dormant durable intent when an edge opens the graph on demand', async () => { + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const activeId = 'durable-a-active'; + const dormantId = 'durable-z-dormant'; + for (const id of [activeId, dormantId]) { + persisted.set(id, { + id, + name: id, + subscribed: true, + synced: false, + sharedMemorySynced: false, + metaSynced: false, + syncScoped: true, + }); + } + + const agentA = await DKGAgent.create({ + name: 'DormantDurableIntentA', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: fixture.subscriptionStore, + contextGraphMembershipStore: fixture.membershipStore, + maxRehydratedContextGraphSubscriptions: 1, + }); + + try { + await agentA.start(); + expect(agentA.getSubscribedContextGraphs().get(activeId)?.syncMode).toBe('always-on'); + expect(agentA.getSubscribedContextGraphs().get(dormantId)).toBeUndefined(); + expect(agentA.getContextGraphSubscriptionRehydrationStatus()).toMatchObject({ + activated: 1, + dormant: 1, + dormantIds: [dormantId], + }); + + // Opening a capped durable row is expressed by the UI as on-demand. + // The agent must recover the stored always-on intent instead of + // downgrading it and later deleting it from the durable store. + const selected = agentA.subscribeToContextGraph(dormantId, { syncMode: 'on-demand' }); + expect(selected.syncMode).toBe('always-on'); + expect(agentA.getSubscribedContextGraphs().get(dormantId)).toMatchObject({ + subscribed: true, + syncMode: 'always-on', + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(persisted.get(dormantId)).toMatchObject({ + id: dormantId, + subscribed: true, + syncScoped: true, + }); + expect(persistedMembers.get(`${dormantId}|node|${agentA.peerId}`)).toMatchObject({ + contextGraphId: dormantId, + status: 'active', + source: 'subscription', + }); + expect(agentA.getContextGraphSubscriptionRehydrationStatus()).toMatchObject({ + activated: 2, + dormant: 0, + dormantIds: [], + }); + } finally { + await agentA.stop().catch(() => {}); + } + + const agentB = await DKGAgent.create({ + name: 'DormantDurableIntentB', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: fixture.subscriptionStore, + contextGraphMembershipStore: fixture.membershipStore, + maxRehydratedContextGraphSubscriptions: 0, + }); + try { + await agentB.start(); + expect(agentB.getSubscribedContextGraphs().get(dormantId)).toMatchObject({ + subscribed: true, + syncMode: 'always-on', + }); + } finally { + await agentB.stop().catch(() => {}); + } + }); + + + it('promotes an existing on-demand selection when the node creates the graph', async () => { + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const contextGraphId = 'selected-then-created-cg'; + const agent = await createAgentWithContextGraphPersistence('OnDemandCreatePromotion', fixture); + + try { + await agent.start(); + agent.subscribeToContextGraph(contextGraphId, { syncMode: 'on-demand' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(persisted.has(contextGraphId)).toBe(false); + + await agent.createContextGraph({ + id: contextGraphId, + name: 'Selected then created', + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(agent.getSubscribedContextGraphs().get(contextGraphId)).toMatchObject({ + subscribed: true, + syncMode: 'always-on', + synced: true, + metaSynced: true, + }); + expect(persisted.get(contextGraphId)).toMatchObject({ + id: contextGraphId, + subscribed: true, + synced: true, + metaSynced: true, + syncScoped: true, + }); + expect(persistedMembers.get(`${contextGraphId}|node|${agent.peerId}`)).toMatchObject({ + contextGraphId, + principalType: 'node', + principalId: agent.peerId, + status: 'active', + source: 'subscription', + }); + } finally { + await agent.stop().catch(() => {}); + } + }); + + + it('persists a Core hosting obligation without persisting its on-demand member intent', async () => { + const persisted = new Map(); + const subscriptionStore = { + loadAll: async () => [...persisted.values()], + save: async (record: any) => { + persisted.set(record.id, { ...record }); + }, + delete: async (contextGraphId: string) => { + persisted.delete(contextGraphId); + }, + }; + const localCgId = 'selected-hosted-cg'; + + const agentA = await DKGAgent.create({ + name: 'OnDemandCoreHostA', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + nodeRole: 'core', + }); + try { + await agentA.start(); + agentA.subscribeToContextGraph(localCgId, { syncMode: 'on-demand' }); + agentA.markContextGraphSubscriptionState(localCgId, { + synced: true, + sharedMemorySynced: true, + metaSynced: true, + }); + (agentA as any).chain.getContextGraphAccessPolicy = async () => 0; + (agentA as any).chain.isContextGraphActiveOnChain = async () => true; + + await (agentA as any).recordCoreHostedPublicCg('14', localCgId); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(agentA.getSubscribedContextGraphs().get(localCgId)).toMatchObject({ + subscribed: true, + syncMode: 'on-demand', + synced: true, + coreHosted: true, + onChainId: '14', + }); + expect(persisted.get(localCgId)).toMatchObject({ + id: localCgId, + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: false, + coreHosted: true, + onChainId: '14', + syncScoped: false, + }); + } finally { + await agentA.stop().catch(() => {}); + } + + const agentB = await DKGAgent.create({ + name: 'OnDemandCoreHostB', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + nodeRole: 'core', + }); + try { + await agentB.start(); + expect(agentB.getSubscribedContextGraphs().get(localCgId)).toMatchObject({ + subscribed: false, + syncMode: 'always-on', + synced: false, + sharedMemorySynced: false, + metaSynced: false, + coreHosted: true, + onChainId: '14', + }); + expect((agentB as any).config.syncContextGraphs ?? []).not.toContain(localCgId); + } finally { + await agentB.stop().catch(() => {}); + } + }); + + it('rehydrates persisted subscriptions without forcing sync scope', async () => { const subscriptionStore = { loadAll: async () => [{ diff --git a/packages/agent/test/core-fills-gap.test.ts b/packages/agent/test/core-fills-gap.test.ts index 495da486df..ba7996bfa1 100644 --- a/packages/agent/test/core-fills-gap.test.ts +++ b/packages/agent/test/core-fills-gap.test.ts @@ -93,7 +93,8 @@ interface AgentInternals { watermarkAfter: number; }>; runVmReconcileSweep(): Promise; - subscribedContextGraphs: Map; + subscribedContextGraphs: Map; + gossipRegistered: Set; vmReconcileDispatcher: { triggerLive: (cg: string) => void; triggerPeriodic: (cg: string) => void; @@ -631,7 +632,7 @@ describe('Phase D — recordCoreHostedPublicCg', () => { expect(((internals as any).recentReconciledUals as { has(key: string): boolean }).has(recentKey)).toBe(false); }); - it('clears VM reconcile state when stale inactive on-chain ids are re-registered', async () => { + it('clears VM reconcile state and promotes durable mode when stale inactive on-chain ids are re-registered', async () => { const internals = await boot(); const localCgId = 'stale-register'; const ownerAddr = (internals.chain as unknown as { signerAddress: string }).signerAddress; @@ -648,9 +649,13 @@ describe('Phase D — recordCoreHostedPublicCg', () => { object: '"5"', graph: contextGraphDataGraphUri(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY), }]); + // This low-level fixture intentionally has no gossip runtime. Model a + // valid already-live on-demand member, including the handler-registration + // invariant that makes subscribeToContextGraph's promotion path idempotent. internals.subscribedContextGraphs.set(localCgId, { - subscribed: true, onChainId: '5', lastReconciledOrdinal: 4, + syncMode: 'on-demand', subscribed: true, onChainId: '5', lastReconciledOrdinal: 4, }); + internals.gossipRegistered.add(localCgId); internals.chain.isContextGraphActiveOnChain = async (id) => id !== 5n; const storageAddr = await internals.chain.getDKGKnowledgeAssetsAddress(); @@ -685,6 +690,7 @@ describe('Phase D — recordCoreHostedPublicCg', () => { const sub = internals.subscribedContextGraphs.get(localCgId); expect(sub?.onChainId).not.toBe('5'); + expect(sub?.syncMode).toBe('always-on'); expect(sub?.lastReconciledOrdinal).toBe(0); expect(recent.has(recentKey)).toBe(false); expect(negativeCache.has(recentKey)).toBe(false); diff --git a/packages/agent/test/discovery-subscription-boundary.test.ts b/packages/agent/test/discovery-subscription-boundary.test.ts index 3ec65ed1e1..daf3ed071c 100644 --- a/packages/agent/test/discovery-subscription-boundary.test.ts +++ b/packages/agent/test/discovery-subscription-boundary.test.ts @@ -652,11 +652,15 @@ describe('Context Graph discovery/subscription boundary', () => { id: 'explicit-local-create', name: 'Explicit Local Create', }); + agent.subscribeToContextGraph('implicit-local-write', { syncMode: 'on-demand' }); + expect(agent.getSubscribedContextGraphs().get('implicit-local-write')?.syncMode).toBe('on-demand'); + expect(persisted.has('implicit-local-write')).toBe(false); await agent.ensureImplicitSharedMemoryContextGraph('implicit-local-write'); await new Promise((resolve) => setTimeout(resolve, 0)); for (const id of ['explicit-local-create', 'implicit-local-write']) { expect(agent.getSubscribedContextGraphs().get(id)?.subscribed).toBe(true); + expect(agent.getSubscribedContextGraphs().get(id)?.syncMode).toBe('always-on'); expect((agent as any).config.syncContextGraphs ?? []).toContain(id); expect((agent as any).gossipRegistered.has(id)).toBe(true); expect(persisted.get(id)).toMatchObject({ subscribed: true, syncScoped: true }); diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index c06e4a3e48..5557b55db5 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises'; import { basename } from 'node:path'; import type { ContextGraphReconcileResult, + ContextGraphSyncMode, RandomSamplingDisabledReason, } from '@origintrail-official/dkg-agent'; import type { JournalReadResult } from '@origintrail-official/dkg-publisher'; @@ -13,6 +14,7 @@ import { } from './finalized-publish-options.js'; import type { RegisterPcaAgentResult } from './pca-confirmation-wire.js'; import { parseRegisterPcaAgentResult } from './pca-confirmation-wire.js'; +import type { CatchupStatusResponse } from './catchup-status-wire.js'; export type { KnowledgeAssetFinalizedPublishOptions } from './finalized-publish-options.js'; @@ -1489,8 +1491,12 @@ export class ApiClient { return this.post('/api/query-remote', { peerId, ...request }); } - async subscribeToContextGraph(contextGraphId: string, options?: { includeSharedMemory?: boolean }): Promise<{ + async subscribeToContextGraph(contextGraphId: string, options: { + includeSharedMemory?: boolean; + syncMode: ContextGraphSyncMode; + }): Promise<{ subscribed: string; + syncMode: ContextGraphSyncMode; catchup?: | { connectedPeers: number; @@ -1548,11 +1554,17 @@ export class ApiClient { } | { status: 'queued'; + includeSharedMemory: boolean; + /** @deprecated Backward-compatible response alias. */ includeWorkspace: boolean; jobId: string; }; }> { - return this.post('/api/context-graph/subscribe', { contextGraphId, includeWorkspace: options?.includeSharedMemory }); + return this.post('/api/context-graph/subscribe', { + contextGraphId, + includeSharedMemory: options.includeSharedMemory, + syncMode: options.syncMode, + }); } /** @@ -1566,6 +1578,7 @@ export class ApiClient { /** @deprecated Use subscribeToContextGraph */ async subscribe(contextGraphId: string, options?: { includeWorkspace?: boolean }): Promise<{ subscribed: string; + syncMode: ContextGraphSyncMode; catchup?: | { connectedPeers: number; @@ -1623,77 +1636,19 @@ export class ApiClient { } | { status: 'queued'; + includeSharedMemory: boolean; + /** @deprecated Backward-compatible response alias. */ includeWorkspace: boolean; jobId: string; }; }> { - return this.subscribeToContextGraph(contextGraphId, { includeSharedMemory: options?.includeWorkspace }); + return this.subscribeToContextGraph(contextGraphId, { + includeSharedMemory: options?.includeWorkspace, + syncMode: 'always-on', + }); } - async catchupStatus(contextGraphId: string): Promise<{ - jobId: string; - contextGraphId: string; - includeWorkspace: boolean; - status: 'queued' | 'running' | 'done' | 'denied' | 'deferred' | 'failed' | 'unreachable'; - queuedAt: number; - startedAt?: number; - finishedAt?: number; - result?: { - connectedPeers: number; - totalPeers?: number; - selectedPeers?: number; - syncCapablePeers: number; - peersTried: number; - peersResponded: number; - peersSucceeded: number; - /** Sync-capable peers skipped because an earlier wave already proved every requested plane. */ - peersNotAttempted?: number; - deferredBackpressure: number; - dataSynced: number; - sharedMemorySynced: number; - denied: boolean; - deniedPeers: number; - diagnostics?: { - noProtocolPeers: number; - durable: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - metaOnlyResponses: number; - verifiedPrivateOnlyResponses?: number; - dataRejectedMissingMeta: number; - rejectedKcs: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - }; - sharedMemory: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - droppedDataTriples: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - }; - }; - }; - error?: string; - }> { + async catchupStatus(contextGraphId: string): Promise { return this.get(`/api/sync/catchup-status?contextGraphId=${encodeURIComponent(contextGraphId)}`); } diff --git a/packages/cli/src/catchup-result-wire.ts b/packages/cli/src/catchup-result-wire.ts new file mode 100644 index 0000000000..470e173af5 --- /dev/null +++ b/packages/cli/src/catchup-result-wire.ts @@ -0,0 +1,90 @@ +/** + * Per-plane clean-completion evidence accumulated across the peers this run + * contacted and safe to expose on the status wire. + */ +export interface CatchupPlaneCompletionEvidence { + verifiedDataPeers: number; + /** Peers that cleanly verified one or more V2 KAs with no public triples. */ + verifiedPrivateOnlyPeers?: number; + emptyPeers: number; + /** + * The metadata-resolved curator cleanly completed this plane while hosting + * the graph and carrying no data at all. + */ + authorityEmptyPeers?: number; + /** + * Peers that answered this plane but whose round did not complete cleanly. + * This prevents one clean-empty response plus one incomplete-empty response + * from being misread as unanimous empty. Pure transport failures remain in + * the plane diagnostics rather than this counter. + */ + incompleteResponders?: number; +} + +/** Stable per-attempt result exposed by the catch-up status API. */ +export interface CatchupJobResult { + connectedPeers: number; + totalPeers?: number; + selectedPeers?: number; + syncCapablePeers: number; + peersTried: number; + /** Peers that reached a responder without collapsing into transport failure. */ + peersResponded: number; + /** Peers whose requested sync round completed cleanly. */ + peersSucceeded: number; + /** Sync-capable peers skipped after an earlier wave proved every requested plane. */ + peersNotAttempted?: number; + /** Context Graph phases deferred by this node's local sync scheduler. */ + deferredBackpressure: number; + dataSynced: number; + sharedMemorySynced: number; + denied: boolean; + deniedPeers: number; + cleanPlaneCompletions?: { + durable: CatchupPlaneCompletionEvidence & { verifiedPrivateOnlyPeers: number }; + sharedMemory: CatchupPlaneCompletionEvidence; + }; + diagnostics?: { + noProtocolPeers: number; + durable: { + fetchedMetaTriples: number; + fetchedDataTriples: number; + insertedMetaTriples: number; + insertedDataTriples: number; + bytesReceived: number; + resumedPhases: number; + timedOutPhases: number; + completedPhases: number; + checkpointAdvances: number; + emptyResponses: number; + metaOnlyResponses: number; + /** Cryptographically verified V2 responses whose public graph is intentionally empty. */ + verifiedPrivateOnlyResponses: number; + dataRejectedMissingMeta: number; + rejectedKcs: number; + failedPeers: number; + failedPhases: number; + deferredBackpressure: number; + deniedPhases?: number; + authorityUnanswered?: boolean; + }; + sharedMemory: { + fetchedMetaTriples: number; + fetchedDataTriples: number; + insertedMetaTriples: number; + insertedDataTriples: number; + bytesReceived: number; + resumedPhases: number; + timedOutPhases: number; + completedPhases: number; + checkpointAdvances: number; + emptyResponses: number; + droppedDataTriples: number; + failedPeers: number; + failedPhases: number; + deferredBackpressure: number; + deniedPhases?: number; + authorityUnanswered?: boolean; + }; + }; +} diff --git a/packages/cli/src/catchup-runner.ts b/packages/cli/src/catchup-runner.ts index 8c06daf29c..6d27276691 100644 --- a/packages/cli/src/catchup-runner.ts +++ b/packages/cli/src/catchup-runner.ts @@ -13,6 +13,15 @@ import { type SyncPeerResolution, } from '@origintrail-official/dkg-agent'; import { PROTOCOL_SYNC } from '@origintrail-official/dkg-core'; +import type { + CatchupJobResult, + CatchupPlaneCompletionEvidence, +} from './catchup-result-wire.js'; + +export type { + CatchupJobResult, + CatchupPlaneCompletionEvidence, +} from './catchup-result-wire.js'; const SYNC_PROTOCOL_CHECK_ATTEMPTS = 3; const SYNC_PROTOCOL_CHECK_DELAY_MS = 500; @@ -20,99 +29,6 @@ const DURABLE_CATCHUP_PHASE_HEADROOM_MS = 1_000; const MIN_DURABLE_CATCHUP_PHASE_BUDGET_MS = 1_000; const DURABLE_CATCHUP_SETTLEMENT_GRACE_MS = 30_000; -export interface CatchupJobResult { - connectedPeers: number; - totalPeers?: number; - selectedPeers?: number; - syncCapablePeers: number; - peersTried: number; - /** - * Subset of `peersTried` whose per-peer sync round reached a responder - * and did not collapse into a transport failure. A responder can still - * time out part-way through, deny access, or serve metadata-only rows; this - * counter exists so daemon status mapping can distinguish "curator offline" - * from "reachable peer answered but did not complete cleanly". - */ - peersResponded: number; - /** - * Subset of `peersTried` whose per-peer sync round finished without a - * transport failure, timeout, or explicit ACL denial, and with either real - * progress or a clean non-metadata-only empty completion. - */ - peersSucceeded: number; - /** - * Sync-capable peers this run deliberately never contacted because an earlier - * wave already proved every requested plane. These are neither failures nor - * successes; they exist so status mapping and operators can tell an - * early-stopped run from a run where peers were unreachable. - */ - peersNotAttempted?: number; - /** Context Graph phases deferred by this node's local sync scheduler. */ - deferredBackpressure: number; - dataSynced: number; - sharedMemorySynced: number; - denied: boolean; - deniedPeers: number; - /** - * Per-plane evidence produced before peer results are aggregated. Aggregate - * diagnostics intentionally retain every timeout/denial for observability, - * but readiness must not let one bad peer mask another peer that completed - * the same plane cleanly and stored verified data. - */ - cleanPlaneCompletions?: { - /** Always carries `verifiedPrivateOnlyPeers`; only the durable plane can produce it. */ - durable: CatchupPlaneCompletionEvidence & { verifiedPrivateOnlyPeers: number }; - sharedMemory: CatchupPlaneCompletionEvidence; - }; - diagnostics?: { - noProtocolPeers: number; - durable: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - metaOnlyResponses: number; - /** Cryptographically verified V2 responses whose public graph is intentionally empty. */ - verifiedPrivateOnlyResponses: number; - dataRejectedMissingMeta: number; - rejectedKcs: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - deniedPhases?: number; - /** A resolvable curator never cleanly answered this plane; see - * `catchupPlaneProvenByUnanimousEmpty`. */ - authorityUnanswered?: boolean; - }; - sharedMemory: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - droppedDataTriples: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - deniedPhases?: number; - /** A resolvable curator never cleanly answered this plane; see - * `catchupPlaneProvenByUnanimousEmpty`. */ - authorityUnanswered?: boolean; - }; - }; -} - export interface CatchupRunRequest { contextGraphId: string; includeSharedMemory: boolean; @@ -513,39 +429,6 @@ export function catchupPlaneCompletedWithoutFailure( return classifyDurableProgress(progress, { complete }).completedWithoutFailure; } -/** Per-plane clean-completion evidence accumulated across the peers this run contacted. */ -export interface CatchupPlaneCompletionEvidence { - verifiedDataPeers: number; - /** Peers that cleanly verified one or more V2 KAs with no public triples. */ - verifiedPrivateOnlyPeers?: number; - emptyPeers: number; - /** - * The metadata-resolved curator cleanly completed this plane while hosting - * the graph and carrying no data at all. See - * {@link catchupPlaneProvenByUnanimousEmpty}. - */ - authorityEmptyPeers?: number; - /** - * Peers that ANSWERED this plane but whose round did not complete cleanly. - * - * Every other field here records what a peer proved. This one records what a - * peer left unresolved, and it exists because the absence of a peer from the - * positive counters is ambiguous: `catchupPeerPlaneEvidence` returns an - * all-zero record for an incomplete round, so a peer that answered EMPTY but - * did not finish paging is indistinguishable from a peer that was never - * contacted. That ambiguity is invisible to the round diagnostics too — an - * explicit `complete: false` is not a transport failure, so it never reaches - * `failedPeers`. - * - * Without it, a round of one clean-empty peer plus one incomplete-empty peer - * reads as unanimously empty. Pure transport failures are deliberately NOT - * counted here: an unreachable stranger is already `failedPeers`, and folding - * it in would pin legitimately empty graphs in a retry loop on a lossy - * network. - */ - incompleteResponders?: number; -} - /** The aggregate per-plane counters a whole-round verdict is allowed to consult. */ export interface CatchupPlaneRoundDiagnostics { fetchedMetaTriples?: number; diff --git a/packages/cli/src/catchup-status-wire.ts b/packages/cli/src/catchup-status-wire.ts new file mode 100644 index 0000000000..97b1748d6a --- /dev/null +++ b/packages/cli/src/catchup-status-wire.ts @@ -0,0 +1,44 @@ +import type { CatchupJobResult } from './catchup-result-wire.js'; +import type { ContextGraphConvergenceSnapshot } from './context-graph-readiness-wire.js'; + +export type CatchupJobState = + | 'queued' + | 'running' + | 'done' + | 'failed' + | 'denied' + /** Local scheduler capacity was unavailable; retry is safe. */ + | 'deferred' + /** + * Peers were reachable, but none delivered every required plane during this + * attempt. This remains distinct from authorization denial and worker error. + */ + | 'unreachable'; + +export interface CatchupConvergenceStatus extends ContextGraphConvergenceSnapshot { + syncMode: 'on-demand' | 'always-on'; + automaticRetryActive: boolean; +} + +/** Shared daemon/client contract for the catch-up status endpoint. */ +export interface CatchupStatusResponse { + jobId: string; + contextGraphId: string; + /** @deprecated Backward-compatible response alias for includeSharedMemory. */ + includeWorkspace: boolean; + includeSharedMemory: boolean; + /** Canonical actionable status, including newer live convergence evidence. */ + status: CatchupJobState; + queuedAt: number; + startedAt?: number; + finishedAt?: number; + result?: CatchupJobResult; + error?: string; + /** Historical runner outcome, emitted only when it differs from status. */ + attempt?: { + status: CatchupJobState; + error?: string; + }; + convergence?: CatchupConvergenceStatus; + completedAfterAttempt?: true; +} diff --git a/packages/cli/src/cli-helpers.ts b/packages/cli/src/cli-helpers.ts index d572fd5544..98ac1c923b 100644 --- a/packages/cli/src/cli-helpers.ts +++ b/packages/cli/src/cli-helpers.ts @@ -202,7 +202,11 @@ function printCatchupStatus(status: Awaited 0) { + console.log(`Missing: ${status.convergence.missing.join(', ')}`); + } + console.log( + `Retry: ${status.convergence.automaticRetryActive ? 'active' : 'inactive'} ` + + `(${status.convergence.syncMode})`, + ); + if (status.completedAfterAttempt) { + console.log('Recovered: a later synchronization completed the selected graph'); + } + } if ( status.result && (status.result.selectedPeers ?? status.result.connectedPeers) >= (status.result.totalPeers ?? status.result.connectedPeers) && diff --git a/packages/cli/src/commands/knowledge.ts b/packages/cli/src/commands/knowledge.ts index 5d1a353e7b..c3177ad917 100644 --- a/packages/cli/src/commands/knowledge.ts +++ b/packages/cli/src/commands/knowledge.ts @@ -342,8 +342,15 @@ program .action(async (contextGraph: string, opts: ActionOpts) => { try { const client = await ApiClient.connect(); - const result = await client.subscribeToContextGraph(contextGraph); + const result = await client.subscribeToContextGraph(contextGraph, { + syncMode: opts.save ? 'always-on' : 'on-demand', + }); console.log(`Subscribed to context graph: ${contextGraph}`); + console.log( + result.syncMode === 'always-on' + ? 'Synchronization mode: always on (restored after restart).' + : 'Synchronization mode: on demand (current node process only).', + ); const catchup = result.catchup; if (catchup) { if ('peersTried' in catchup) { diff --git a/packages/cli/src/context-graph-readiness-wire.ts b/packages/cli/src/context-graph-readiness-wire.ts new file mode 100644 index 0000000000..fa924c8d9d --- /dev/null +++ b/packages/cli/src/context-graph-readiness-wire.ts @@ -0,0 +1,20 @@ +/** Serializable convergence planes shared by daemon and client contracts. */ +export type ContextGraphConvergencePlane = 'metadata' | 'durable' | 'sharedMemory'; + +/** Pure wire shape for the independently verified state of one context graph. */ +export interface ContextGraphConvergenceSnapshot { + state: 'pending' | 'partial' | 'complete'; + required: { + metadata: true; + durable: true; + sharedMemory: boolean; + }; + verified: { + metadata: boolean; + durable: boolean; + sharedMemory: boolean; + }; + missing: ContextGraphConvergencePlane[]; + readinessUpdatedAt?: number; + observedAt: number; +} diff --git a/packages/cli/src/context-graph-readiness.ts b/packages/cli/src/context-graph-readiness.ts index 91aab03a66..5050f23900 100644 --- a/packages/cli/src/context-graph-readiness.ts +++ b/packages/cli/src/context-graph-readiness.ts @@ -13,8 +13,16 @@ import { type CatchupJobResult, type CatchupPlaneCompletionEvidence, } from './catchup-runner.js'; +import type { + ContextGraphConvergencePlane, + ContextGraphConvergenceSnapshot, +} from './context-graph-readiness-wire.js'; export { catchupPlaneCompletedWithoutFailure } from './catchup-runner.js'; +export type { + ContextGraphConvergencePlane, + ContextGraphConvergenceSnapshot, +} from './context-graph-readiness-wire.js'; export const CONTEXT_GRAPH_READINESS_VERSION = 1; @@ -42,6 +50,195 @@ export interface ContextGraphReadinessPatch { sharedMemoryVerified: boolean; } +/** Narrow capability required to decide whether local CG metadata is trusted. */ +export interface ContextGraphMetadataAuthority { + isCuratorOf?: (contextGraphId: string) => Promise; + hasConfirmedMetaState: ( + contextGraphId: string, + options?: { rejectUnregisteredPlaceholder?: boolean }, + ) => Promise; +} + +export type ContextGraphReadinessPlanes = Omit< + ContextGraphConvergenceSnapshot, + 'observedAt' +>; + +interface ResolvedReadinessPlanes { + state: ContextGraphReadinessPlanes['state']; + metadataVerified: boolean; + durableVerified: boolean; + sharedMemoryVerified: boolean; + missingMetadata: boolean; + missingDurable: boolean; + missingRequestedSharedMemory: boolean; + readinessUpdatedAt?: number; +} + +function hasCurrentReadinessProvenance( + readiness: ContextGraphReadinessProvenance, +): boolean { + return readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; +} + +function describeResolvedReadinessPlanes(input: { + hasConfirmedMeta: boolean; + includeSharedMemory: boolean; + durableEvidence: boolean; + sharedMemoryEvidence: boolean; + readinessUpdatedAt?: number; +}): ResolvedReadinessPlanes { + const metadataVerified = input.hasConfirmedMeta; + const durableVerified = metadataVerified && input.durableEvidence; + const sharedMemoryVerified = metadataVerified && input.sharedMemoryEvidence; + const missingMetadata = !metadataVerified; + const missingDurable = !durableVerified; + const missingRequestedSharedMemory = + input.includeSharedMemory && !sharedMemoryVerified; + + const anyVerified = metadataVerified || durableVerified || + (input.includeSharedMemory && sharedMemoryVerified); + const complete = !missingMetadata && !missingDurable && + !missingRequestedSharedMemory; + + return { + state: complete ? 'complete' : anyVerified ? 'partial' : 'pending', + metadataVerified, + durableVerified, + sharedMemoryVerified, + missingMetadata, + missingDurable, + missingRequestedSharedMemory, + ...(input.readinessUpdatedAt !== undefined + ? { readinessUpdatedAt: input.readinessUpdatedAt } + : {}), + }; +} + +function renderReadinessPlanes( + planes: ResolvedReadinessPlanes, + includeSharedMemory: boolean, +): ContextGraphReadinessPlanes { + const missing: ContextGraphConvergencePlane[] = []; + if (planes.missingMetadata) missing.push('metadata'); + if (planes.missingDurable) missing.push('durable'); + if (planes.missingRequestedSharedMemory) missing.push('sharedMemory'); + return { + state: planes.state, + required: { + metadata: true, + durable: true, + sharedMemory: includeSharedMemory, + }, + verified: { + metadata: planes.metadataVerified, + durable: planes.durableVerified, + sharedMemory: planes.sharedMemoryVerified, + }, + missing, + ...(planes.readinessUpdatedAt !== undefined + ? { readinessUpdatedAt: planes.readinessUpdatedAt } + : {}), + }; +} + +function resolvePersistedReadinessPlanes(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ResolvedReadinessPlanes { + const currentReadinessProvenance = hasCurrentReadinessProvenance(input.readiness); + return describeResolvedReadinessPlanes({ + hasConfirmedMeta: input.hasConfirmedMeta, + includeSharedMemory: input.includeSharedMemory, + durableEvidence: + currentReadinessProvenance && input.readiness.durableVerified, + sharedMemoryEvidence: + currentReadinessProvenance && input.readiness.sharedMemoryVerified, + ...(currentReadinessProvenance + ? { readinessUpdatedAt: input.readiness.updatedAt } + : {}), + }); +} + +/** + * Canonical interpretation of persisted readiness as independently verified + * metadata, durable VM, and optional SWM planes. Every caller that needs to + * decide whether a selected graph is ready must go through this helper so + * metadata loss and readiness-version changes cannot drift between paths. + */ +export function describeReadinessPlanes(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ContextGraphReadinessPlanes { + return renderReadinessPlanes( + resolvePersistedReadinessPlanes(input), + input.includeSharedMemory, + ); +} + +function resolveCatchupPlaneEvidence(input: { + readinessBeforeCatchup: ContextGraphReadinessProvenance; + durableReadyThisRun: boolean; + sharedMemoryReadyThisRun: boolean; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ResolvedReadinessPlanes { + const currentReadinessProvenance = hasCurrentReadinessProvenance( + input.readinessBeforeCatchup, + ); + return describeResolvedReadinessPlanes({ + hasConfirmedMeta: input.hasConfirmedMeta, + includeSharedMemory: input.includeSharedMemory, + durableEvidence: + (currentReadinessProvenance && + input.readinessBeforeCatchup.durableVerified) || + input.durableReadyThisRun, + sharedMemoryEvidence: + (currentReadinessProvenance && + input.readinessBeforeCatchup.sharedMemoryVerified) || + input.sharedMemoryReadyThisRun, + ...(currentReadinessProvenance + ? { readinessUpdatedAt: input.readinessBeforeCatchup.updatedAt } + : {}), + }); +} + +/** + * Merge current-run completion evidence with persisted provenance without + * pretending the merged value was itself read from storage. + */ +export function combineCatchupPlaneEvidence(input: { + readinessBeforeCatchup: ContextGraphReadinessProvenance; + durableReadyThisRun: boolean; + sharedMemoryReadyThisRun: boolean; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ContextGraphReadinessPlanes { + return renderReadinessPlanes( + resolveCatchupPlaneEvidence(input), + input.includeSharedMemory, + ); +} + +/** + * Describe the live, per-plane convergence of one selected context graph. + * A VM/SWM proof is only effective while authoritative CG metadata is local; + * stale provenance must not make a graph look complete after metadata loss. + */ +export function describeContextGraphConvergence(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; + observedAt?: number; +}): ContextGraphConvergenceSnapshot { + return { + ...describeReadinessPlanes(input), + observedAt: input.observedAt ?? Date.now(), + }; +} + export interface MissingMetadataReadinessPatches { statePatch: ContextGraphSubscriptionStatePatch; readinessPatch: ContextGraphReadinessPatch; @@ -73,17 +270,12 @@ export function classifyExistingContextGraphReadiness(input: { statePatch?: ContextGraphSubscriptionStatePatch; readinessPatch?: ContextGraphReadinessPatch; } { - const currentReadinessProvenance = - input.readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; - const overallReadinessVerified = - input.readiness.durableVerified || input.readiness.sharedMemoryVerified; - const requestedPlanesVerified = - currentReadinessProvenance && - overallReadinessVerified && - (!input.includeSharedMemory || input.readiness.sharedMemoryVerified); + const currentReadinessProvenance = hasCurrentReadinessProvenance( + input.readiness, + ); + const planes = resolvePersistedReadinessPlanes(input); const alreadyReady = - input.hasConfirmedMeta && - requestedPlanesVerified && + planes.state === 'complete' && input.subscription.synced === true && (!input.includeSharedMemory || input.subscription.sharedMemorySynced === true); @@ -109,16 +301,13 @@ export function classifyExistingContextGraphReadiness(input: { }; } - const durableVerified = - currentReadinessProvenance && input.readiness.durableVerified; - const sharedMemoryVerified = - currentReadinessProvenance && input.readiness.sharedMemoryVerified; - const overallVerified = durableVerified || sharedMemoryVerified; + const durableVerified = planes.durableVerified; + const sharedMemoryVerified = planes.sharedMemoryVerified; const statePatch = - input.subscription.synced !== overallVerified || + input.subscription.synced !== durableVerified || input.subscription.sharedMemorySynced !== sharedMemoryVerified ? { - synced: overallVerified, + synced: durableVerified, sharedMemorySynced: sharedMemoryVerified, metaSynced: true, pendingMeta: false, @@ -270,7 +459,7 @@ function catchupPlaneReadinessThisRun(input: { } export interface ContextGraphCatchupReadinessClassification { - jobStatus: 'done' | 'failed' | 'denied' | 'unreachable'; + jobStatus: 'done' | 'failed' | 'denied' | 'unreachable' | 'deferred'; error?: string; statePatch?: ContextGraphSubscriptionStatePatch; readinessPatch?: ContextGraphReadinessPatch; @@ -281,12 +470,96 @@ export interface ContextGraphCatchupReadinessClassification { }; } +function deferredCatchupClassification(): ContextGraphCatchupReadinessClassification { + return { + jobStatus: 'deferred', + error: 'Sync deferred by local scheduler backpressure; retry when capacity is available.', + }; +} + /** - * Canonical policy for converting one catch-up result into externally visible - * subscription readiness. The HTTP route gathers live metadata and applies - * the returned patches; all readiness decisions remain in this pure function. + * Backpressure is scoped to the planes requested by this view. A durable-only + * projection of a broad execution must not inherit shared-memory deferral. + * Legacy results without plane diagnostics fall back to the aggregate counter + * so rolling-upgrade callers remain fail-closed. */ -export function classifyContextGraphCatchupReadiness(input: { +function requestedCatchupBackpressure( + result: CatchupJobResult, + includeSharedMemory: boolean, +): number { + if (includeSharedMemory) return result.deferredBackpressure; + return result.diagnostics?.durable.deferredBackpressure + ?? result.deferredBackpressure; +} + +interface ContextGraphCatchupReadinessInput { + result: CatchupJobResult; + includeSharedMemory: boolean; + readinessBeforeCatchup: ContextGraphReadinessProvenance; +} + +export interface ContextGraphCatchupMetadata { + hasConfirmedMeta: boolean; + isPrivate: boolean; +} + +export type ContextGraphCatchupReadinessPlan = + | { + kind: 'settled'; + classification: ContextGraphCatchupReadinessClassification; + } + | { + kind: 'metadata-required'; + finalize: ( + metadata: ContextGraphCatchupMetadata, + ) => ContextGraphCatchupReadinessClassification; + }; + +/** + * Canonical two-phase classifier for one catch-up attempt. The plan owns the + * decision to request chain-backed metadata, so orchestration never mirrors + * classifier branches merely to avoid an unnecessary metadata read. + */ +export function planContextGraphCatchupReadiness( + input: ContextGraphCatchupReadinessInput, +): ContextGraphCatchupReadinessPlan { + const deferredBackpressure = requestedCatchupBackpressure( + input.result, + input.includeSharedMemory, + ); + if (deferredBackpressure > 0 && !input.result.denied) { + return { + kind: 'settled', + classification: deferredCatchupClassification(), + }; + } + + const finalize = ( + metadata: ContextGraphCatchupMetadata, + ): ContextGraphCatchupReadinessClassification => { + const classification = classifyContextGraphCatchupReadinessWithoutBackpressure({ + ...input, + ...metadata, + }); + // Pure ACL denial keeps its specific status. When denial coexists with + // clean requested-plane progress, local deferral still prevents that + // partial round from becoming successful or freezing readiness. + return deferredBackpressure > 0 && classification.jobStatus === 'done' + ? deferredCatchupClassification() + : classification; + }; + + if (!catchupResultHasCleanResponse(input.result)) { + return { + kind: 'settled', + classification: finalize({ hasConfirmedMeta: false, isPrivate: false }), + }; + } + + return { kind: 'metadata-required', finalize }; +} + +function classifyContextGraphCatchupReadinessWithoutBackpressure(input: { result: CatchupJobResult; includeSharedMemory: boolean; hasConfirmedMeta: boolean; @@ -343,41 +616,41 @@ export function classifyContextGraphCatchupReadiness(input: { currentReadinessProvenance && input.readinessBeforeCatchup.durableVerified; const sharedMemoryVerifiedBefore = currentReadinessProvenance && input.readinessBeforeCatchup.sharedMemoryVerified; - const durableVerified = durableVerifiedBefore || durableReadyThisRun; - const sharedMemoryVerified = sharedMemoryVerifiedBefore || sharedMemoryReadyThisRun; + const planes = resolveCatchupPlaneEvidence({ + readinessBeforeCatchup: input.readinessBeforeCatchup, + durableReadyThisRun, + sharedMemoryReadyThisRun, + includeSharedMemory: input.includeSharedMemory, + hasConfirmedMeta: input.hasConfirmedMeta, + }); + const durableVerified = planes.durableVerified; + const sharedMemoryVerified = planes.sharedMemoryVerified; // What this run is allowed to FREEZE, as opposed to what it reports. These // diverge only for a unanimous-empty verdict, which stays re-derived per run // so that a wrong empty verdict cannot become permanent. const durableVerifiedPersisted = durableVerifiedBefore || durableThisRun.persistable; const sharedMemoryVerifiedPersisted = sharedMemoryVerifiedBefore || sharedMemoryThisRun.persistable; - // `subscription.synced` is a SECOND persisted readiness bit, living outside - // the provenance store and consumed by callers that never see this job's - // result — `contextGraphRowIsWritable` treats `subscribed && synced` as - // writable. It must therefore carry the same verdict as `readinessPatch`, - // not the transient one. The pre-catch-up path already assumes they agree: - // it derives `synced` from the persisted provenance and patches the row - // back into line, so letting them diverge here would be corrected away on - // the next pass anyway — after a window in which the graph looked writable. - const overallVerifiedPersisted = durableVerifiedPersisted || sharedMemoryVerifiedPersisted; - const overallVerified = durableVerified || sharedMemoryVerified; - const missingGraphProof = !overallVerified; - const missingRequestedSharedMemory = - input.includeSharedMemory && !sharedMemoryVerified; + const missingDurable = planes.missingDurable; + const missingRequestedSharedMemory = planes.missingRequestedSharedMemory; const madeIncompleteProgress = (durableDataProgress && !durableReadyThisRun) || (sharedMemoryProgress && !sharedMemoryReadyThisRun); let jobStatus: ContextGraphCatchupReadinessClassification['jobStatus'] = 'done'; let error: string | undefined; - if (missingGraphProof || missingRequestedSharedMemory) { + if (missingDurable || missingRequestedSharedMemory) { jobStatus = 'unreachable'; if (madeIncompleteProgress) { error = 'Verified data was inserted, but catch-up did not complete without a timeout or failed phase. The incomplete plane remains unready; retry once the network is healthier.'; - } else if (input.isPrivate && missingGraphProof) { - error = 'No authorized context-graph peer delivered verified durable or shared-memory data — empty or metadata-only responses cannot prove a private graph is fully synchronized, and the curator may be offline.'; - } else if (input.isPrivate) { + } else if (input.isPrivate && missingDurable && sharedMemoryVerified) { + error = 'Shared-memory catch-up completed, but no authorized peer delivered a verified durable VM snapshot. The selected graph remains incomplete.'; + } else if (input.isPrivate && missingDurable) { + error = 'No authorized context-graph peer delivered verified durable VM data — empty or metadata-only responses cannot prove a private graph is fully synchronized, and the curator may be offline.'; + } else if (input.isPrivate && missingRequestedSharedMemory) { error = 'Durable context-graph data synchronized, but shared-memory catch-up did not complete. Retry to finish shared-memory synchronization.'; + } else if (missingDurable && sharedMemoryVerified) { + error = 'Shared-memory catch-up completed, but durable VM catch-up did not complete. The selected graph remains incomplete.'; } else { error = 'Context-graph catch-up did not complete cleanly for every requested data plane. Retry once the network is healthier.'; } @@ -387,7 +660,10 @@ export function classifyContextGraphCatchupReadiness(input: { jobStatus, error, statePatch: { - synced: overallVerifiedPersisted, + // `synced` is a second persisted VM-readiness bit used by write + // preflight, so it must match durable provenance rather than transient + // empty-round evidence or independently verified SWM. + synced: durableVerifiedPersisted, sharedMemorySynced: sharedMemoryVerifiedPersisted, metaSynced: true, pendingMeta: false, @@ -498,6 +774,24 @@ async function withContextGraphReadinessMutationLock( } } +/** + * Require authoritative metadata for a readiness decision. Only a locally + * curated graph may trust its own pre-registration definition; remote graphs + * must reject the legacy unregistered placeholder shape. + */ +export async function hasAuthoritativeContextGraphMetadata(input: { + agent: ContextGraphMetadataAuthority; + contextGraphId: string; +}): Promise { + const locallyCurated = typeof input.agent.isCuratorOf === 'function' + ? await input.agent.isCuratorOf(input.contextGraphId).catch(() => false) + : false; + return input.agent.hasConfirmedMetaState( + input.contextGraphId, + { rejectUnregisteredPlaceholder: !locallyCurated }, + ).catch(() => false); +} + /** * Revalidate live metadata and invalidate subscription/provenance together. * Returns false when authoritative metadata arrived before this reset acquired @@ -512,13 +806,10 @@ export async function resetContextGraphReadinessForMissingMetadata(input: { if (!contextGraphId) return false; return withContextGraphReadinessMutationLock(input.agent, contextGraphId, async () => { - const locallyCurated = typeof input.agent.isCuratorOf === 'function' - ? await input.agent.isCuratorOf(contextGraphId).catch(() => false) - : false; - const hasConfirmedMeta = await input.agent.hasConfirmedMetaState( + const hasConfirmedMeta = await hasAuthoritativeContextGraphMetadata({ + agent: input.agent, contextGraphId, - { rejectUnregisteredPlaceholder: !locallyCurated }, - ).catch(() => false); + }); if (hasConfirmedMeta) return false; const patches = missingMetadataReadinessPatches(); diff --git a/packages/cli/src/daemon/catchup-status-response.ts b/packages/cli/src/daemon/catchup-status-response.ts new file mode 100644 index 0000000000..b639cd3f81 --- /dev/null +++ b/packages/cli/src/daemon/catchup-status-response.ts @@ -0,0 +1,99 @@ +import type { + CatchupConvergenceStatus, + CatchupStatusResponse, +} from '../catchup-status-wire.js'; +import { + describeContextGraphConvergence, + hasAuthoritativeContextGraphMetadata, + readContextGraphReadiness, + type ContextGraphMetadataAuthority, + type ContextGraphReadinessStore, +} from '../context-graph-readiness.js'; +import type { CatchupJob } from './types.js'; + +export type { + CatchupConvergenceStatus, + CatchupStatusResponse, +} from '../catchup-status-wire.js'; + +/** Project one immutable catch-up attempt plus live convergence onto the wire. */ +export function toCatchupStatusResponse( + job: CatchupJob, + convergence?: CatchupConvergenceStatus, +): CatchupStatusResponse { + const completedAfterAttempt = convergence?.state === 'complete' && + convergence.readinessUpdatedAt !== undefined && + job.finishedAt !== undefined && + convergence.readinessUpdatedAt > job.finishedAt && + (job.status === 'failed' || + job.status === 'deferred' || + job.status === 'unreachable'); + const invalidatedAfterAttempt = job.status === 'done' && + convergence !== undefined && + convergence.state !== 'complete' && + convergence.readinessUpdatedAt !== undefined && + job.finishedAt !== undefined && + convergence.readinessUpdatedAt > job.finishedAt; + const status = completedAfterAttempt + ? 'done' + : invalidatedAfterAttempt + ? 'unreachable' + : job.status; + const { status: attemptStatus, error: attemptError, ...attemptFields } = job; + return { + ...attemptFields, + contextGraphId: job.contextGraphId, + // `includeWorkspace` remains wire-only for pre-rename clients. + includeWorkspace: job.includeSharedMemory, + status, + ...(status === attemptStatus && attemptError ? { error: attemptError } : {}), + ...(status !== attemptStatus + ? { + attempt: { + status: attemptStatus, + ...(attemptError ? { error: attemptError } : {}), + }, + } + : {}), + ...(convergence ? { convergence } : {}), + ...(completedAfterAttempt ? { completedAfterAttempt: true } : {}), + }; +} + +export interface CatchupStatusAgent extends ContextGraphMetadataAuthority { + getSubscribedContextGraphs(): ReadonlyMap; +} + +/** Load live convergence and apply the canonical status projection together. */ +export async function loadCatchupStatusResponse(input: { + job: CatchupJob; + agent: CatchupStatusAgent; + readinessStore: Partial; + observedAt?: number; +}): Promise { + const subscription = input.agent.getSubscribedContextGraphs() + .get(input.job.contextGraphId); + const hasConfirmedMeta = await hasAuthoritativeContextGraphMetadata({ + agent: input.agent, + contextGraphId: input.job.contextGraphId, + }); + const convergence: CatchupConvergenceStatus = { + ...describeContextGraphConvergence({ + readiness: readContextGraphReadiness( + input.readinessStore, + input.job.contextGraphId, + ), + includeSharedMemory: input.job.includeSharedMemory, + hasConfirmedMeta, + ...(input.observedAt === undefined + ? {} + : { observedAt: input.observedAt }), + }), + syncMode: subscription?.syncMode ?? 'always-on', + automaticRetryActive: subscription?.subscribed === true, + }; + return toCatchupStatusResponse(input.job, convergence); +} diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts new file mode 100644 index 0000000000..7a9f68f907 --- /dev/null +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -0,0 +1,396 @@ +import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg-node-ui'; +import type { CatchupRunner } from '../catchup-runner.js'; +import type { CatchupJobResult } from '../catchup-result-wire.js'; +import { + planContextGraphCatchupReadiness, + type ContextGraphCatchupReadinessClassification, +} from '../context-graph-readiness.js'; +import type { + CatchupCoordinator, + CatchupJob, + CatchupScope, + CatchupTracker, +} from './types.js'; + +export interface ContextGraphCatchupCoordinatorEffects { + runner: Pick; + readReadiness: (contextGraphId: string) => ContextGraphReadinessProvenance; + hasConfirmedMeta: (contextGraphId: string) => Promise; + isPrivate: (contextGraphId: string) => Promise; + writeReadiness: ( + contextGraphId: string, + patch: { durableVerified: boolean; sharedMemoryVerified: boolean }, + ) => void; + markSubscriptionState: ( + contextGraphId: string, + patch: { + synced: boolean; + sharedMemorySynced: boolean; + metaSynced: boolean; + pendingMeta: boolean; + }, + ) => void; + emitProjectSynced: ( + contextGraphId: string, + payload: { + dataSynced: number; + sharedMemorySynced: number; + verifiedPrivateOnlyResponses: number; + }, + ) => void; + now?: () => number; + createJobId?: () => string; + trace?: (message: string) => void; +} + +type CatchupResultClassification = ContextGraphCatchupReadinessClassification; + +export class ContextGraphCatchupCoordinatorService { + private readonly now: () => number; + private readonly createJobId: () => string; + private readonly inFlightByContextGraph = new Map(); + + constructor( + private readonly tracker: CatchupTracker, + private readonly effects: ContextGraphCatchupCoordinatorEffects, + ) { + this.now = effects.now ?? Date.now; + this.createJobId = effects.createJobId ?? (() => + `${this.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`); + } + + /** Reuse active work while preserving each caller's immutable plane scope. */ + coalesceActive(input: { + contextGraphId: string; + includeSharedMemory: boolean; + }): CatchupJob | undefined { + const coordinator = this.inFlightByContextGraph.get(input.contextGraphId); + if (!coordinator) return undefined; + if (!this.hasActiveJob(coordinator)) return undefined; + + const requestedScope = this.toScope(input.includeSharedMemory); + const existingJobId = this.jobIdForScope(coordinator, requestedScope); + if (existingJobId) { + const existingJob = this.tracker.jobs.get(existingJobId); + if (existingJob) return this.markLatest(existingJob); + } + + if (requestedScope === 'durable') { + if (coordinator.initialScope !== 'durable-and-shared-memory') return undefined; + const durable = this.createJob(input.contextGraphId, requestedScope); + coordinator.durableJobId = durable.jobId; + return this.markLatest(durable); + } + + if (coordinator.initialScope !== 'durable') return undefined; + const full = this.createJob(input.contextGraphId, requestedScope); + coordinator.fullJobId = full.jobId; + return this.markLatest(full); + } + + /** Start one detached serialized worker for a fresh per-CG catch-up. */ + start(input: { + contextGraphId: string; + includeSharedMemory: boolean; + readinessBeforeCatchup: ContextGraphReadinessProvenance; + }): CatchupJob { + const scope = this.toScope(input.includeSharedMemory); + const job = this.createJob(input.contextGraphId, scope); + const coordinator: CatchupCoordinator = { + contextGraphId: input.contextGraphId, + initialScope: scope, + ...(scope === 'durable' + ? { durableJobId: job.jobId } + : { fullJobId: job.jobId }), + }; + this.inFlightByContextGraph.set(input.contextGraphId, coordinator); + this.pruneCompletedJobs(); + void this.run(coordinator, input.readinessBeforeCatchup); + return this.markLatest(job); + } + + private toScope(includeSharedMemory: boolean): CatchupScope { + return includeSharedMemory ? 'durable-and-shared-memory' : 'durable'; + } + + private createJob(contextGraphId: string, scope: CatchupScope): CatchupJob { + const job: CatchupJob = { + jobId: this.createJobId(), + contextGraphId, + includeSharedMemory: scope === 'durable-and-shared-memory', + status: 'queued', + queuedAt: this.now(), + }; + this.tracker.jobs.set(job.jobId, job); + return job; + } + + private markLatest(job: CatchupJob): CatchupJob { + this.tracker.latestByContextGraph.set(job.contextGraphId, job.jobId); + return job; + } + + private jobIdForScope( + coordinator: CatchupCoordinator, + scope: CatchupScope, + ): string | undefined { + return scope === 'durable' + ? coordinator.durableJobId + : coordinator.fullJobId; + } + + private hasActiveJob(coordinator: CatchupCoordinator): boolean { + return [coordinator.durableJobId, coordinator.fullJobId].some((jobId) => { + if (!jobId) return false; + const job = this.tracker.jobs.get(jobId); + return job?.status === 'queued' || job?.status === 'running'; + }); + } + + private pruneCompletedJobs(): void { + const activeJobIds = new Set(); + for (const coordinator of this.inFlightByContextGraph.values()) { + if (coordinator.durableJobId) activeJobIds.add(coordinator.durableJobId); + if (coordinator.fullJobId) activeJobIds.add(coordinator.fullJobId); + } + while (this.tracker.jobs.size > 100) { + let oldest: CatchupJob | undefined; + for (const candidate of this.tracker.jobs.values()) { + if (activeJobIds.has(candidate.jobId)) continue; + if (!oldest || candidate.queuedAt < oldest.queuedAt) oldest = candidate; + } + if (!oldest) break; + this.tracker.jobs.delete(oldest.jobId); + if ( + this.tracker.latestByContextGraph.get(oldest.contextGraphId) === oldest.jobId + ) { + this.tracker.latestByContextGraph.delete(oldest.contextGraphId); + } + } + } + + private async run( + coordinator: CatchupCoordinator, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + try { + if (coordinator.initialScope === 'durable') { + await this.runDurableFirst(coordinator, readinessBeforeCatchup); + } else { + await this.runFullFirst(coordinator, readinessBeforeCatchup); + } + } finally { + if (this.inFlightByContextGraph.get(coordinator.contextGraphId) === coordinator) { + this.inFlightByContextGraph.delete(coordinator.contextGraphId); + } + } + } + + private async runDurableFirst( + coordinator: CatchupCoordinator, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + const durable = coordinator.durableJobId + ? this.tracker.jobs.get(coordinator.durableJobId) + : undefined; + if (!durable || durable.status !== 'queued') return; + try { + const attempt = await this.runAttempt(durable, readinessBeforeCatchup); + if (attempt.status === 'denied') { + this.settleFullSlotFrom(coordinator, durable); + return; + } + } catch (error) { + this.settleThrownJob(durable, error); + this.settleFullSlotFrom(coordinator, durable); + return; + } + + const full = coordinator.fullJobId + ? this.tracker.jobs.get(coordinator.fullJobId) + : undefined; + if (!full || full.status !== 'queued') return; + try { + await this.runAttempt(full, this.effects.readReadiness(coordinator.contextGraphId)); + } catch (error) { + this.settleThrownJob(full, error); + } + } + + private async runFullFirst( + coordinator: CatchupCoordinator, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + const full = coordinator.fullJobId + ? this.tracker.jobs.get(coordinator.fullJobId) + : undefined; + if (!full || full.status !== 'queued') return; + try { + const attempt = await this.runAttempt(full, readinessBeforeCatchup); + await this.settleDurableSlot( + coordinator, + full, + attempt.result, + readinessBeforeCatchup, + this.classificationHasEffects(attempt.classification), + ); + } catch (error) { + this.settleThrownJob(full, error); + this.settleDurableSlotFrom(coordinator, full); + } + } + + private settleThrownJob(job: CatchupJob, error: unknown): void { + job.error = error instanceof Error ? error.message : String(error); + job.status = 'failed'; + job.finishedAt = this.now(); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} threw: ${job.error}`, + ); + } + + private async runAttempt( + job: CatchupJob, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise<{ + result: CatchupJobResult; + status: CatchupJob['status']; + classification: CatchupResultClassification; + }> { + job.status = 'running'; + job.startedAt ??= this.now(); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} started`, + ); + const result = await this.effects.runner.run({ + contextGraphId: job.contextGraphId, + includeSharedMemory: job.includeSharedMemory, + }); + const classification = await this.classifyResult( + job, + result, + readinessBeforeCatchup, + ); + this.applyExecutionEffects(job.contextGraphId, classification); + this.settleClassifiedJob(job, result, classification); + return { result, status: job.status, classification }; + } + + private classificationHasEffects( + classification: CatchupResultClassification, + ): boolean { + return classification.readinessPatch !== undefined || + classification.statePatch !== undefined || + classification.eventPayload !== undefined; + } + + private async classifyResult( + job: CatchupJob, + result: CatchupJobResult, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + const plan = planContextGraphCatchupReadiness({ + result, + includeSharedMemory: job.includeSharedMemory, + readinessBeforeCatchup, + }); + if (plan.kind === 'settled') return plan.classification; + + const hasConfirmedMeta = await this.effects.hasConfirmedMeta(job.contextGraphId); + const isPrivate = hasConfirmedMeta + ? await this.effects.isPrivate(job.contextGraphId) + : false; + return plan.finalize({ hasConfirmedMeta, isPrivate }); + } + + private applyExecutionEffects( + contextGraphId: string, + classification: CatchupResultClassification, + ): void { + if (classification.readinessPatch) { + this.effects.writeReadiness(contextGraphId, classification.readinessPatch); + } + if (classification.statePatch) { + this.effects.markSubscriptionState(contextGraphId, classification.statePatch); + } + if (classification.eventPayload) { + this.effects.emitProjectSynced(contextGraphId, classification.eventPayload); + } + } + + private settleClassifiedJob( + job: CatchupJob, + result: CatchupJobResult, + classification: CatchupResultClassification, + ): void { + job.result = result; + job.status = classification.jobStatus; + job.error = classification.error; + job.finishedAt = this.now(); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} status=${job.status} ` + + `peers=${result.peersTried}/${result.syncCapablePeers} ` + + `connected=${result.totalPeers ?? result.connectedPeers} ` + + `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, + ); + } + + private settleQueuedJobFrom(targetJobId: string, source: CatchupJob): void { + const target = this.tracker.jobs.get(targetJobId); + if (!target || target.status !== 'queued') return; + target.status = source.status; + target.error = source.error; + target.result = source.result; + target.startedAt = source.startedAt; + target.finishedAt = this.now(); + } + + private async settleDurableSlot( + coordinator: CatchupCoordinator, + full: CatchupJob, + result: CatchupJobResult, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + sourceAppliedEffects: boolean, + ): Promise { + const durable = coordinator.durableJobId + ? this.tracker.jobs.get(coordinator.durableJobId) + : undefined; + if (!durable || durable.status !== 'queued') return; + durable.status = 'running'; + durable.startedAt = full.startedAt; + try { + const classification = await this.classifyResult( + durable, + result, + readinessBeforeCatchup, + ); + // Full success already persisted every plane. When the full attempt had + // no effects (for example SWM-only local deferral), the independently + // successful durable slot owns its scoped readiness and event instead. + if (!sourceAppliedEffects) { + this.applyExecutionEffects(durable.contextGraphId, classification); + } + this.settleClassifiedJob(durable, result, classification); + } catch (error) { + this.settleThrownJob(durable, error); + } + } + + private settleDurableSlotFrom( + coordinator: CatchupCoordinator, + source: CatchupJob, + ): void { + if (coordinator.durableJobId) { + this.settleQueuedJobFrom(coordinator.durableJobId, source); + } + } + + private settleFullSlotFrom( + coordinator: CatchupCoordinator, + source: CatchupJob, + ): void { + if (coordinator.fullJobId) { + this.settleQueuedJobFrom(coordinator.fullJobId, source); + } + } +} diff --git a/packages/cli/src/daemon/context-graph-catchup-route-adapter.ts b/packages/cli/src/daemon/context-graph-catchup-route-adapter.ts new file mode 100644 index 0000000000..2a57430ed5 --- /dev/null +++ b/packages/cli/src/daemon/context-graph-catchup-route-adapter.ts @@ -0,0 +1,49 @@ +import type { DKGAgent } from '@origintrail-official/dkg-agent'; +import { DKGEvent } from '@origintrail-official/dkg-core'; +import type { DashboardDB } from '@origintrail-official/dkg-node-ui'; +import type { CatchupRunner } from '../catchup-runner.js'; +import { + hasAuthoritativeContextGraphMetadata, + readContextGraphReadiness, + writeContextGraphReadiness, +} from '../context-graph-readiness.js'; +import { ContextGraphCatchupCoordinatorService } from './context-graph-catchup-coordinator.js'; +import type { CatchupTracker } from './types.js'; + +/** + * Bind daemon-owned persistence, agent, and event effects once at the route + * boundary. The subscribe route only chooses a scope and delegates + * start/coalescing; readiness classification and side effects stay behind the + * coordinator's narrow API. + */ +export function createContextGraphCatchupRouteAdapter(input: { + tracker: CatchupTracker; + runner: Pick; + readinessStore: DashboardDB; + agent: DKGAgent; + trace?: (message: string) => void; +}): ContextGraphCatchupCoordinatorService { + return new ContextGraphCatchupCoordinatorService(input.tracker, { + runner: input.runner, + readReadiness: (contextGraphId) => + readContextGraphReadiness(input.readinessStore, contextGraphId), + hasConfirmedMeta: (contextGraphId) => + hasAuthoritativeContextGraphMetadata({ + agent: input.agent, + contextGraphId, + }), + isPrivate: (contextGraphId) => + input.agent.isPrivateContextGraph(contextGraphId).catch(() => true), + writeReadiness: (contextGraphId, patch) => + writeContextGraphReadiness(input.readinessStore, contextGraphId, patch), + markSubscriptionState: (contextGraphId, patch) => + input.agent.markContextGraphSubscriptionState(contextGraphId, patch), + emitProjectSynced: (contextGraphId, payload) => { + input.agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { + contextGraphId, + ...payload, + }); + }, + ...(input.trace ? { trace: input.trace } : {}), + }); +} diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index 54e6365506..327405b481 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -146,7 +146,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from './types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index b0fae05696..a22da2c129 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -203,15 +203,16 @@ import { DkgClient } from '@origintrail-official/dkg-mcp/client'; // the project's tsconfig (`noUnusedLocals` is off). import { daemonState, + DEBUG_SYNC_TRACE, resolveStandaloneInstall, resolveAutoUpdatePollingMode, type CorsAllowlist, } from './state.js'; +import { createContextGraphCatchupRouteAdapter } from './context-graph-catchup-route-adapter.js'; import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from './types.js'; import { type MarkItDownTarget, @@ -1072,13 +1073,13 @@ export async function bootstrapConfiguredContextGraphs(input: { input.log( `Context graph "${contextGraphId}" setup failed: ${err instanceof Error ? err.message : String(err)} — will discover via sync/gossip`, ); - input.agent.subscribeToContextGraph(contextGraphId); + input.agent.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); } continue; } const existing = input.agent.getSubscribedContextGraphs().get(contextGraphId); - input.agent.subscribeToContextGraph(contextGraphId); + input.agent.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); let hasAuthoritativeMetadata = false; let locallyCurated = false; @@ -3460,6 +3461,15 @@ export async function runDaemonInner( let corsAllowed: CorsAllowlist = "*"; daemonState.catchupRunner = createCatchupRunner(agent); + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner, + readinessStore: dashDb, + agent, + ...(DEBUG_SYNC_TRACE + ? { trace: (message: string) => console.log(message) } + : {}), + }); const server = createServer(async (req, res) => { try { @@ -3655,6 +3665,7 @@ export async function runDaemonInner( nodeVersion, nodeCommit, catchupTracker, + catchupCoordinator, extractionRegistry, fileStore, extractionStatus, diff --git a/packages/cli/src/daemon/routes/agent-chat.ts b/packages/cli/src/daemon/routes/agent-chat.ts index 44cdeb0e0f..a24e469d79 100644 --- a/packages/cli/src/daemon/routes/agent-chat.ts +++ b/packages/cli/src/daemon/routes/agent-chat.ts @@ -166,7 +166,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 947e27e15e..a898cfa725 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -65,7 +65,7 @@ import { VmReconcileQueueFullError, VmReconcileUnavailableError, } from '@origintrail-official/dkg-agent'; -import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri, SYSTEM_CONTEXT_GRAPHS } from '@origintrail-official/dkg-core'; +import { computeNetworkId, createOperationContext, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri, SYSTEM_CONTEXT_GRAPHS } from '@origintrail-official/dkg-core'; import { findReservedSubjectPrefix, isSkolemizedUri } from '@origintrail-official/dkg-publisher'; import { DashboardDB, @@ -114,9 +114,8 @@ import { import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; import { - catchupResultHasCleanResponse, - classifyContextGraphCatchupReadiness, classifyExistingContextGraphReadiness, + hasAuthoritativeContextGraphMetadata, readContextGraphReadiness, writeContextGraphReadiness, } from '../../context-graph-readiness.js'; @@ -159,8 +158,6 @@ import { DkgClient } from '@origintrail-official/dkg-mcp/client'; // them all without explicit imports. Unused ones are tolerated by // the project's tsconfig (`noUnusedLocals` is off). import { - daemonState, - DEBUG_SYNC_TRACE, resolveAutoUpdateEnabled, type CorsAllowlist, } from '../state.js'; @@ -168,7 +165,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, @@ -482,6 +478,7 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise false); + await hasAuthoritativeContextGraphMetadata({ agent, contextGraphId }); const existingReadiness = classifyExistingContextGraphReadiness({ subscription: existingSub, readiness: readinessBeforeCatchup, @@ -1753,13 +1773,16 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise 100) { - let oldestId: string | undefined; - let oldestQueuedAt = Number.POSITIVE_INFINITY; - for (const [id, entry] of catchupTracker.jobs.entries()) { - if (entry.queuedAt < oldestQueuedAt) { - oldestQueuedAt = entry.queuedAt; - oldestId = id; - } - } - if (!oldestId) break; - const removed = catchupTracker.jobs.get(oldestId); - catchupTracker.jobs.delete(oldestId); - if ( - removed && - catchupTracker.latestByContextGraph.get(removed.contextGraphId) === oldestId - ) { - catchupTracker.latestByContextGraph.delete(removed.contextGraphId); - } - } - - void (async () => { - job.status = "running"; - job.startedAt = Date.now(); - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} started`); - try { - const result = await daemonState.catchupRunner!.run({ - contextGraphId: contextGraphId, - includeSharedMemory: shouldSyncSharedMemory, - }); - job.result = result; - // Local scheduler pressure cut the round short. An incomplete round has - // no readiness to inspect and must never finalize the subscription, so - // short-circuit the whole classification path and report a distinct - // retryable status. A remote denial still wins: waiting for local - // capacity will never clear it. - if (result.deferredBackpressure > 0 && !result.denied) { - job.status = "deferred"; - job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} deferred by local scheduler: ${result.deferredBackpressure}`); - } else { - const inspectReadiness = catchupResultHasCleanResponse(result); - const hasConfirmedMeta = inspectReadiness - ? await agent.hasConfirmedMetaState(contextGraphId).catch(() => false) - : false; - const isPrivate = hasConfirmedMeta - ? await agent.isPrivateContextGraph(contextGraphId).catch(() => true) - : false; - const classification = classifyContextGraphCatchupReadiness({ - result, - includeSharedMemory: shouldSyncSharedMemory, - hasConfirmedMeta, - isPrivate, - readinessBeforeCatchup, - }); - - job.status = classification.jobStatus; - job.error = classification.error; - if (classification.readinessPatch) { - writeContextGraphReadiness( - dashDb, - contextGraphId, - classification.readinessPatch, - ); - } - if (classification.statePatch) { - agent.markContextGraphSubscriptionState( - contextGraphId, - classification.statePatch, - ); - } - if (classification.eventPayload) { - agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { - contextGraphId, - ...classification.eventPayload, - }); - } - - // Denial took precedence above, but a denied-yet-still-deferred round - // is likewise incomplete: never let it settle as a successful "done". - if (job.status === "done" && result.deferredBackpressure > 0) { - job.status = "deferred"; - job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; - } - } - - if (DEBUG_SYNC_TRACE) { - if (job.status === 'denied') { - console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} denied by remote peer(s): ${result.deniedPeers}`); - } - console.log( - `[catchup] job=${jobId} contextGraph=${contextGraphId} status=${job.status} ` + - `peers=${result.peersTried}/${result.syncCapablePeers} ` + - `connected=${result.totalPeers ?? result.connectedPeers} ` + - `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, - ); - } - } catch (err) { - job.error = err instanceof Error ? err.message : String(err); - job.status = "failed"; - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} threw: ${job.error}`); - } finally { - job.finishedAt = Date.now(); - } - })(); + includeSharedMemory: shouldSyncSharedMemory, + readinessBeforeCatchup, + }); return jsonResponse(res, 200, { subscribed: contextGraphId, + syncMode: effectiveSyncMode, catchup: { status: "queued", + includeSharedMemory: shouldSyncSharedMemory, includeWorkspace: shouldSyncSharedMemory, - jobId, + jobId: job.jobId, }, }); } diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index dbad8dd72d..bf3626947f 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -24,6 +24,7 @@ import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; import type { CatchupTracker } from '../types.js'; import type { RoutePlugin } from '../plugin-api.js'; import type { AdmissionStatsView } from '../http-utils.js'; +import type { ContextGraphCatchupCoordinatorService } from '../context-graph-catchup-coordinator.js'; export type MemoryGraphLayer = 'wm' | 'swm' | 'vm'; @@ -73,6 +74,8 @@ export interface RequestContext { nodeVersion: string; nodeCommit: string; catchupTracker: CatchupTracker; + /** Lifecycle-owned coordinator; active coalescing state never leaks into status storage. */ + catchupCoordinator: ContextGraphCatchupCoordinatorService; extractionRegistry: ExtractionPipelineRegistry; fileStore: FileStore; extractionStatus: Map; diff --git a/packages/cli/src/daemon/routes/epcis.ts b/packages/cli/src/daemon/routes/epcis.ts index ca1ba118fa..bcc45664b5 100644 --- a/packages/cli/src/daemon/routes/epcis.ts +++ b/packages/cli/src/daemon/routes/epcis.ts @@ -154,7 +154,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/local-agents.ts b/packages/cli/src/daemon/routes/local-agents.ts index 42c47b1903..f1be067cbc 100644 --- a/packages/cli/src/daemon/routes/local-agents.ts +++ b/packages/cli/src/daemon/routes/local-agents.ts @@ -153,7 +153,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/memory.ts b/packages/cli/src/daemon/routes/memory.ts index fe336927b6..d01d190bf8 100644 --- a/packages/cli/src/daemon/routes/memory.ts +++ b/packages/cli/src/daemon/routes/memory.ts @@ -171,7 +171,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/openclaw.ts b/packages/cli/src/daemon/routes/openclaw.ts index 8592d2cdf3..74f3a4a8cc 100644 --- a/packages/cli/src/daemon/routes/openclaw.ts +++ b/packages/cli/src/daemon/routes/openclaw.ts @@ -153,7 +153,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/publisher.ts b/packages/cli/src/daemon/routes/publisher.ts index 28101fe6cc..1ea479668e 100644 --- a/packages/cli/src/daemon/routes/publisher.ts +++ b/packages/cli/src/daemon/routes/publisher.ts @@ -154,7 +154,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/query.ts b/packages/cli/src/daemon/routes/query.ts index e4d14e3e90..64d26b237e 100644 --- a/packages/cli/src/daemon/routes/query.ts +++ b/packages/cli/src/daemon/routes/query.ts @@ -166,8 +166,8 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; +import { loadCatchupStatusResponse } from '../catchup-status-response.js'; import { type MarkItDownTarget, manifestRepoRoot, @@ -1033,7 +1033,12 @@ export async function handleQueryRoutes(ctx: RequestContext): Promise { }); } - return jsonResponse(res, 200, toCatchupStatusResponse(job)); + const response = await loadCatchupStatusResponse({ + job, + agent, + readinessStore: dashDb, + }); + return jsonResponse(res, 200, response); } // POST /api/verify diff --git a/packages/cli/src/daemon/routes/status.ts b/packages/cli/src/daemon/routes/status.ts index 83b1d2644a..5fcb039530 100644 --- a/packages/cli/src/daemon/routes/status.ts +++ b/packages/cli/src/daemon/routes/status.ts @@ -160,7 +160,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/types.ts b/packages/cli/src/daemon/types.ts index 2da37e8b51..c908f9bfdf 100644 --- a/packages/cli/src/daemon/types.ts +++ b/packages/cli/src/daemon/types.ts @@ -3,29 +3,14 @@ // Pure type/interface declarations used across the daemon sub-modules. import type { CatchupJobResult } from '../catchup-runner.js'; +import type { CatchupJobState } from '../catchup-status-wire.js'; -export type CatchupJobState = - | "queued" - | "running" - | "done" - | "failed" - | "denied" - /** Local scheduler capacity was unavailable; retry is safe. */ - | "deferred" - /** - * Catchup completed but no peer could deliver the CG content within - * the run — every per-peer sync round either failed or returned - * nothing while no responder explicitly denied access. Distinct from - * `denied` (curator refused) and `failed` (the worker itself threw) - * so the UI can render targeted copy + a "send signed join request" - * CTA without misclassifying slow public CGs as denied. - */ - | "unreachable"; +export type { CatchupJobState } from '../catchup-status-wire.js'; export interface CatchupJob { jobId: string; contextGraphId: string; - includeWorkspace: boolean; // kept for wire compat; semantically "includeSharedMemory" + includeSharedMemory: boolean; status: CatchupJobState; queuedAt: number; startedAt?: number; @@ -34,15 +19,22 @@ export interface CatchupJob { error?: string; } +export type CatchupScope = 'durable' | 'durable-and-shared-memory'; + +/** + * One serialized per-CG catch-up has exactly the two scopes the product + * exposes. When durable starts first, a later full request occupies the full + * slot and runs second. When full starts first, a later durable request + * occupies the durable slot and is settled from that full result. + */ +export interface CatchupCoordinator { + contextGraphId: string; + initialScope: CatchupScope; + durableJobId?: string; + fullJobId?: string; +} + export interface CatchupTracker { jobs: Map; latestByContextGraph: Map; } - -export function toCatchupStatusResponse(job: CatchupJob) { - return { - ...job, - contextGraphId: job.contextGraphId, - includeSharedMemory: job.includeWorkspace, - }; -} diff --git a/packages/cli/test/api-client.test.ts b/packages/cli/test/api-client.test.ts index b84db84d94..0d64d8aaa7 100644 --- a/packages/cli/test/api-client.test.ts +++ b/packages/cli/test/api-client.test.ts @@ -459,6 +459,44 @@ describe('ApiClient', () => { }); describe('POST endpoints', () => { + it('subscribeToContextGraph() forwards explicit sync lifetime', async () => { + const { fetch, calls } = createTrackingFetch({ + ok: true, + status: 200, + body: { subscribed: 'cg-selected', syncMode: 'on-demand' }, + }); + globalThis.fetch = fetch; + + await client.subscribeToContextGraph('cg-selected', { + includeSharedMemory: true, + syncMode: 'on-demand', + }); + + expect(calls[0].url).toBe(`http://127.0.0.1:${PORT}/api/context-graph/subscribe`); + expect(JSON.parse(calls[0].opts.body as string)).toEqual({ + contextGraphId: 'cg-selected', + includeSharedMemory: true, + syncMode: 'on-demand', + }); + }); + + it('subscribe() maps its deprecated workspace option to the canonical request key', async () => { + const { fetch, calls } = createTrackingFetch({ + ok: true, + status: 200, + body: { subscribed: 'cg-legacy', syncMode: 'always-on' }, + }); + globalThis.fetch = fetch; + + await client.subscribe('cg-legacy', { includeWorkspace: true }); + + expect(JSON.parse(calls[0].opts.body as string)).toEqual({ + contextGraphId: 'cg-legacy', + includeSharedMemory: true, + syncMode: 'always-on', + }); + }); + it('sendChat() sends correct body', async () => { const { fetch, calls } = createTrackingFetch({ ok: true, status: 200, body: { delivered: true } }); globalThis.fetch = fetch; diff --git a/packages/cli/test/catchup-status-cli.test.ts b/packages/cli/test/catchup-status-cli.test.ts new file mode 100644 index 0000000000..491a174498 --- /dev/null +++ b/packages/cli/test/catchup-status-cli.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ApiClient } from '../src/api-client.js'; +import type { CatchupStatusResponse } from '../src/catchup-status-wire.js'; +import { + printCatchupStatus, + runCatchupStatusCommand, +} from '../src/cli-helpers.js'; + +const recoveredStatus: CatchupStatusResponse = { + jobId: 'job-recovered', + contextGraphId: 'cg-recovered', + includeWorkspace: false, + includeSharedMemory: true, + status: 'done', + queuedAt: 1, + startedAt: 2, + finishedAt: 3, + attempt: { + status: 'unreachable', + error: 'durable VM missing', + }, + convergence: { + state: 'complete', + required: { metadata: true, durable: true, sharedMemory: true }, + verified: { metadata: true, durable: true, sharedMemory: true }, + missing: [], + readinessUpdatedAt: 4, + observedAt: 5, + syncMode: 'on-demand', + automaticRetryActive: true, + }, + completedAfterAttempt: true, +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('catch-up status CLI', () => { + it('renders actionable recovery with historical attempt diagnostics', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + + printCatchupStatus(recoveredStatus); + + const lines = log.mock.calls.map(([line]) => String(line)); + expect(lines).toContain('Status: done'); + expect(lines).toContain('Last attempt: unreachable'); + expect(lines).toContain('Attempt error: durable VM missing'); + expect(lines).toContain('Shared Memory: enabled'); + expect(lines).toContain('Convergence: complete'); + expect(lines).toContain('Recovered: a later synchronization completed the selected graph'); + }); + + it('stops watch mode on the canonical actionable status', async () => { + const catchupStatus = vi.fn().mockResolvedValue(recoveredStatus); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ catchupStatus } as ApiClient); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'clear').mockImplementation(() => {}); + + await runCatchupStatusCommand('cg-recovered', { watch: true, interval: 1 }); + + expect(catchupStatus).toHaveBeenCalledTimes(1); + expect(catchupStatus).toHaveBeenCalledWith('cg-recovered'); + }); +}); diff --git a/packages/cli/test/catchup-status-convergence-route.test.ts b/packages/cli/test/catchup-status-convergence-route.test.ts new file mode 100644 index 0000000000..75951beb1d --- /dev/null +++ b/packages/cli/test/catchup-status-convergence-route.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from 'vitest'; +import { + lossyPublicEmptyResult, + privateSharedMemoryOnlyResult, +} from './helpers/context-graph-catchup-fixtures.js'; +import { ContextGraphSubscribeRouteHarness } from './helpers/context-graph-subscribe-route-harness.js'; + +async function runIncompletePrivateAttempt( + options: { strictHasConfirmedMeta?: boolean; locallyCurated?: boolean } = {}, +) { + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + strictHasConfirmedMeta: options.strictHasConfirmedMeta, + locallyCurated: options.locallyCurated, + isPrivate: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: () => privateSharedMemoryOnlyResult(), + }); + const response = await harness.postSubscribe(); + const jobId = response.body.catchup.jobId as string; + const job = await harness.waitForJob(jobId); + return { harness, jobId, job }; +} + +describe('catch-up status live convergence', () => { + it('preserves a lossy public empty-round success without persisted completion', async () => { + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + isPrivate: false, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: () => lossyPublicEmptyResult(), + }); + try { + const response = await harness.postSubscribe({ includeSharedMemory: false }); + const jobId = response.body.catchup.jobId as string; + const job = await harness.waitForJob(jobId); + expect(job).toMatchObject({ status: 'done' }); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'partial', + verified: { + metadata: true, + durable: false, + sharedMemory: false, + }, + missing: ['durable'], + }, + }); + } finally { + await harness.close(); + } + }); + + it('keeps a failed attempt failed when complete readiness predates the job', async () => { + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: true, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + readiness: { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: 1, + }, + runner: () => { + throw new Error('foreground attempt failed'); + }, + }); + try { + const response = await harness.postSubscribe(); + const jobId = response.body.catchup.jobId as string; + const job = await harness.waitForJob(jobId); + expect(job?.status).toBe('failed'); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'failed', + error: 'foreground attempt failed', + convergence: { + state: 'complete', + readinessUpdatedAt: 1, + }, + }); + } finally { + await harness.close(); + } + }); + + it('reports completion after persisted readiness recovers a failed attempt', async () => { + const { harness, jobId, job } = await runIncompletePrivateAttempt(); + try { + expect(job?.status).toBe('unreachable'); + harness.setCompleteReadiness((job?.finishedAt ?? Date.now()) + 1); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'done', + attempt: { + status: 'unreachable', + error: expect.stringContaining('durable VM'), + }, + completedAfterAttempt: true, + convergence: { + state: 'complete', + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + }, + }); + } finally { + await harness.close(); + } + }); + + it('rejects legacy placeholder metadata instead of rewriting failure to done', async () => { + const { harness, jobId, job } = await runIncompletePrivateAttempt({ + strictHasConfirmedMeta: false, + locallyCurated: false, + }); + try { + expect(job?.status).toBe('unreachable'); + harness.setCompleteReadiness((job?.finishedAt ?? Date.now()) + 1); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'unreachable', + convergence: { + state: 'pending', + verified: { + metadata: false, + durable: false, + sharedMemory: false, + }, + missing: ['metadata', 'durable', 'sharedMemory'], + }, + }); + expect(harness.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: true, + }); + } finally { + await harness.close(); + } + }); + + it('preserves the local-curator metadata exception used by bootstrap', async () => { + const { harness, jobId, job } = await runIncompletePrivateAttempt({ + strictHasConfirmedMeta: false, + locallyCurated: true, + }); + try { + harness.setCompleteReadiness((job?.finishedAt ?? Date.now()) + 1); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'done', + completedAfterAttempt: true, + convergence: { state: 'complete', missing: [] }, + }); + expect(harness.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: false, + }); + } finally { + await harness.close(); + } + }); +}); diff --git a/packages/cli/test/catchup-status-response.test.ts b/packages/cli/test/catchup-status-response.test.ts new file mode 100644 index 0000000000..25ff76f54d --- /dev/null +++ b/packages/cli/test/catchup-status-response.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from 'vitest'; +import { + toCatchupStatusResponse, + type CatchupConvergenceStatus, +} from '../src/daemon/catchup-status-response.js'; +import type { CatchupJob } from '../src/daemon/types.js'; + +const completeConvergence: CatchupConvergenceStatus = { + state: 'complete', + required: { + metadata: true, + durable: true, + sharedMemory: true, + }, + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + readinessUpdatedAt: 10, + observedAt: 20, + syncMode: 'on-demand', + automaticRetryActive: true, +}; + +function job(status: CatchupJob['status']): CatchupJob { + return { + jobId: 'job-1', + contextGraphId: 'cg-1', + includeSharedMemory: true, + status, + queuedAt: 1, + startedAt: 2, + finishedAt: 3, + error: `${status} attempt`, + }; +} + +describe('catch-up status response', () => { + it('keeps the legacy workspace name at the response boundary only', () => { + const attempt = job('done'); + + expect(attempt).toHaveProperty('includeSharedMemory', true); + expect(attempt).not.toHaveProperty('includeWorkspace'); + expect(toCatchupStatusResponse(attempt)).toMatchObject({ + includeSharedMemory: true, + includeWorkspace: true, + }); + }); + + it('reports live completion while preserving a failed attempt as diagnostics', () => { + expect(toCatchupStatusResponse(job('failed'), completeConvergence)).toMatchObject({ + status: 'done', + attempt: { + status: 'failed', + error: 'failed attempt', + }, + completedAfterAttempt: true, + convergence: completeConvergence, + }); + expect(toCatchupStatusResponse(job('failed'), completeConvergence)) + .not.toHaveProperty('error'); + }); + + it('reports newer live completion while preserving a deferred attempt as diagnostics', () => { + expect(toCatchupStatusResponse(job('deferred'), completeConvergence)).toMatchObject({ + status: 'done', + attempt: { + status: 'deferred', + error: 'deferred attempt', + }, + completedAfterAttempt: true, + convergence: completeConvergence, + }); + expect(toCatchupStatusResponse(job('deferred'), completeConvergence)) + .not.toHaveProperty('error'); + }); + + it('does not hide a failed attempt behind readiness that predates it', () => { + const staleConvergence = { ...completeConvergence, readinessUpdatedAt: 3 }; + + expect(toCatchupStatusResponse(job('failed'), staleConvergence)).toMatchObject({ + status: 'failed', + error: 'failed attempt', + convergence: staleConvergence, + }); + expect(toCatchupStatusResponse(job('failed'), staleConvergence)) + .not.toHaveProperty('completedAfterAttempt'); + expect(toCatchupStatusResponse(job('failed'), staleConvergence)) + .not.toHaveProperty('attempt'); + }); + + it('never lets historical readiness override a current authorization denial', () => { + expect(toCatchupStatusResponse(job('denied'), completeConvergence)).toMatchObject({ + status: 'denied', + error: 'denied attempt', + convergence: completeConvergence, + }); + }); + + it('downgrades the actionable status when a completed attempt loses convergence', () => { + const invalidatedConvergence = { + ...completeConvergence, + state: 'pending' as const, + verified: { metadata: false, durable: false, sharedMemory: false }, + missing: ['metadata', 'durable', 'sharedMemory'] as const, + }; + + expect(toCatchupStatusResponse(job('done'), invalidatedConvergence)).toMatchObject({ + status: 'unreachable', + attempt: { status: 'done' }, + convergence: invalidatedConvergence, + }); + }); + + it('preserves a successful attempt when incomplete convergence is not a newer invalidation', () => { + const nonPersistedConvergence = { + ...completeConvergence, + state: 'partial' as const, + verified: { metadata: true, durable: false, sharedMemory: false }, + missing: ['durable', 'sharedMemory'] as const, + readinessUpdatedAt: undefined, + }; + + expect(toCatchupStatusResponse(job('done'), nonPersistedConvergence)) + .toMatchObject({ + status: 'done', + convergence: nonPersistedConvergence, + }); + expect(toCatchupStatusResponse(job('done'), nonPersistedConvergence)) + .not.toHaveProperty('attempt'); + }); +}); diff --git a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts new file mode 100644 index 0000000000..118f6c68a6 --- /dev/null +++ b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts @@ -0,0 +1,344 @@ +import { describe, expect, it } from 'vitest'; +import { + privateDataOnlyResult, + publicDurableAndSharedMemoryResult, + publicDurableWithSharedMemoryBackpressureResult, +} from './helpers/context-graph-catchup-fixtures.js'; +import { ContextGraphSubscribeRouteHarness } from './helpers/context-graph-subscribe-route-harness.js'; + +function deferred(): { + promise: Promise; + resolve: () => void; +} { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +describe('context graph catch-up route coalescing', () => { + it('makes a reused broad job latest after broad, narrow, broad requests', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async () => { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return publicDurableAndSharedMemoryResult(); + }, + }); + + try { + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await firstRunStarted.promise; + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: narrow.body.catchup.jobId, + includeSharedMemory: false, + }); + + const reusedBroad = await harness.postSubscribe({ includeSharedMemory: true }); + expect(reusedBroad.body.catchup.jobId).toBe(broad.body.catchup.jobId); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: broad.body.catchup.jobId, + includeSharedMemory: true, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('makes a reused narrow job latest after narrow, upgrade, narrow requests', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async (_request, callNumber) => { + if (callNumber === 1) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return privateDataOnlyResult(); + } + return publicDurableAndSharedMemoryResult(); + }, + }); + + try { + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + await firstRunStarted.promise; + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: broad.body.catchup.jobId, + includeSharedMemory: true, + }); + + const reusedNarrow = await harness.postSubscribe({ includeSharedMemory: false }); + expect(reusedNarrow.body.catchup.jobId).toBe(narrow.body.catchup.jobId); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: narrow.body.catchup.jobId, + includeSharedMemory: false, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('serially upgrades an active VM-only request with a distinct VM plus SWM job', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async (_request, callNumber) => { + if (callNumber === 1) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return privateDataOnlyResult(); + } + return publicDurableAndSharedMemoryResult(); + }, + }); + + try { + const base = await harness.postSubscribe({ includeSharedMemory: false }); + await firstRunStarted.promise; + const upgrade = await harness.postSubscribe({ includeSharedMemory: true }); + + expect(upgrade.body.catchup).toMatchObject({ + status: 'queued', + includeSharedMemory: true, + }); + expect(upgrade.body.catchup.jobId).not.toBe(base.body.catchup.jobId); + expect(harness.runCalls).toBe(1); + expect(harness.getJob(upgrade.body.catchup.jobId)?.status).toBe('queued'); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: upgrade.body.catchup.jobId, + status: 'queued', + includeSharedMemory: true, + includeWorkspace: true, + }); + + releaseFirstRun.resolve(); + const baseJob = await harness.waitForJob(base.body.catchup.jobId); + const upgradeJob = await harness.waitForJob(upgrade.body.catchup.jobId); + + expect(harness.runRequests).toEqual([ + { + contextGraphId: harness.contextGraphId, + includeSharedMemory: false, + }, + { + contextGraphId: harness.contextGraphId, + includeSharedMemory: true, + }, + ]); + expect(baseJob).toMatchObject({ includeSharedMemory: false, status: 'done' }); + expect(upgradeJob).toMatchObject({ includeSharedMemory: true, status: 'done' }); + await expect(harness.getStatus(base.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + await expect(harness.getStatus(upgrade.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: true }, + missing: [], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('projects broad in-flight work onto a distinct VM-only contract', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async () => { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return privateDataOnlyResult(); + }, + }); + + try { + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await firstRunStarted.promise; + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + + expect(narrow.body.catchup).toMatchObject({ + status: 'queued', + includeSharedMemory: false, + }); + expect(narrow.body.catchup.jobId).not.toBe(broad.body.catchup.jobId); + expect(harness.runCalls).toBe(1); + + releaseFirstRun.resolve(); + const broadJob = await harness.waitForJob(broad.body.catchup.jobId); + const narrowJob = await harness.waitForJob(narrow.body.catchup.jobId); + + expect(harness.runCalls).toBe(1); + expect(broadJob).toMatchObject({ + includeSharedMemory: true, + status: 'unreachable', + }); + expect(narrowJob).toMatchObject({ + includeSharedMemory: false, + status: 'done', + }); + await expect(harness.getStatus(narrow.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + await expect(harness.getStatus(broad.body.catchup.jobId)).resolves.toMatchObject({ + status: 'unreachable', + convergence: { + state: 'partial', + required: { sharedMemory: true }, + missing: ['sharedMemory'], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('keeps a VM projection done when only the broad SWM plane is backpressured', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async () => { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return publicDurableWithSharedMemoryBackpressureResult(); + }, + }); + + try { + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await firstRunStarted.promise; + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + + releaseFirstRun.resolve(); + const broadJob = await harness.waitForJob(broad.body.catchup.jobId); + const narrowJob = await harness.waitForJob(narrow.body.catchup.jobId); + + expect(harness.runCalls).toBe(1); + expect(broadJob).toMatchObject({ status: 'deferred' }); + expect(narrowJob).toMatchObject({ status: 'done' }); + await expect(harness.getStatus(narrow.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('keeps VM-only success stable when the serialized SWM upgrade fails', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async (_request, callNumber) => { + if (callNumber === 1) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + } + return privateDataOnlyResult(); + }, + }); + + try { + const base = await harness.postSubscribe({ includeSharedMemory: false }); + await firstRunStarted.promise; + const upgrade = await harness.postSubscribe({ includeSharedMemory: true }); + releaseFirstRun.resolve(); + + const baseJob = await harness.waitForJob(base.body.catchup.jobId); + const upgradeJob = await harness.waitForJob(upgrade.body.catchup.jobId); + expect(baseJob).toMatchObject({ includeSharedMemory: false, status: 'done' }); + expect(upgradeJob).toMatchObject({ + includeSharedMemory: true, + status: 'unreachable', + error: expect.stringContaining('requested data plane'), + }); + await expect(harness.getStatus(base.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + await expect(harness.getStatus(upgrade.body.catchup.jobId)).resolves.toMatchObject({ + status: 'unreachable', + convergence: { + state: 'partial', + required: { sharedMemory: true }, + missing: ['sharedMemory'], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); +}); diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts new file mode 100644 index 0000000000..bf52629a02 --- /dev/null +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -0,0 +1,438 @@ +import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg-node-ui'; +import { describe, expect, it, vi } from 'vitest'; +import type { CatchupJobResult } from '../src/catchup-runner.js'; +import { ContextGraphCatchupCoordinatorService } from '../src/daemon/context-graph-catchup-coordinator.js'; +import type { CatchupJob, CatchupTracker } from '../src/daemon/types.js'; +import { + cleanEmptyResult, + privateDataOnlyResult, + publicDurableAndSharedMemoryResult, + publicDurableWithSharedMemoryBackpressureResult, +} from './helpers/context-graph-catchup-fixtures.js'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function waitForJob(job: CatchupJob): Promise { + for (let attempt = 0; attempt < 50 && !job.finishedAt; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + if (!job.finishedAt) throw new Error(`catch-up job ${job.jobId} did not settle`); +} + +function deniedResult() { + const result = cleanEmptyResult(); + result.denied = true; + result.deniedPeers = 1; + result.peersSucceeded = 0; + if (!result.cleanPlaneCompletions || !result.diagnostics) { + throw new Error('completion evidence missing'); + } + result.cleanPlaneCompletions.durable.emptyPeers = 0; + result.cleanPlaneCompletions.sharedMemory.emptyPeers = 0; + result.diagnostics.durable.emptyResponses = 0; + result.diagnostics.sharedMemory.emptyResponses = 0; + return result; +} + +function coordinatorFixture(options: { + failUpgrade?: boolean; + baseOutcome?: 'success' | 'throw' | 'denied'; + blockBroadBase?: boolean; + result?: CatchupJobResult; +} = {}) { + const tracker: CatchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; + let readiness: ContextGraphReadinessProvenance = { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }; + let sequence = 0; + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + let runNumber = 0; + const run = vi.fn(async (request: { includeSharedMemory: boolean }) => { + runNumber += 1; + if (runNumber === 1 && (!request.includeSharedMemory || options.blockBroadBase)) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + if (options.baseOutcome === 'throw') { + throw new Error('base attempt failed'); + } + if (options.baseOutcome === 'denied') return deniedResult(); + if (options.result) return options.result; + return privateDataOnlyResult(); + } + if (options.result) return options.result; + return options.failUpgrade + ? privateDataOnlyResult() + : publicDurableAndSharedMemoryResult(); + }); + const writeReadiness = vi.fn((_contextGraphId: string, patch: { + durableVerified: boolean; + sharedMemoryVerified: boolean; + }) => { + readiness = { ...readiness, ...patch, updatedAt: readiness.updatedAt + 1 }; + }); + const markSubscriptionState = vi.fn(); + const hasConfirmedMeta = vi.fn(async () => true); + const isPrivate = vi.fn(async () => false); + const service = new ContextGraphCatchupCoordinatorService(tracker, { + runner: { run }, + readReadiness: () => readiness, + hasConfirmedMeta, + isPrivate, + writeReadiness, + markSubscriptionState, + emitProjectSynced: vi.fn(), + createJobId: () => `job-${++sequence}`, + }); + return { + tracker, + service, + run, + firstRunStarted, + releaseFirstRun, + writeReadiness, + markSubscriptionState, + hasConfirmedMeta, + isPrivate, + }; +} + +describe('ContextGraphCatchupCoordinatorService', () => { + it('does not load metadata for pure local deferral', async () => { + const fixture = coordinatorFixture({ + result: publicDurableWithSharedMemoryBackpressureResult(), + }); + const full = fixture.service.start({ + contextGraphId: 'cg:pure-deferral', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + + await waitForJob(full); + + expect(full.status).toBe('deferred'); + expect(fixture.hasConfirmedMeta).not.toHaveBeenCalled(); + expect(fixture.isPrivate).not.toHaveBeenCalled(); + }); + + it('settles a durable slot from broad work without inheriting SWM backpressure', async () => { + const fixture = coordinatorFixture({ + blockBroadBase: true, + result: publicDurableWithSharedMemoryBackpressureResult(), + }); + const full = fixture.service.start({ + contextGraphId: 'cg:scope-backpressure', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const durable = fixture.service.coalesceActive({ + contextGraphId: 'cg:scope-backpressure', + includeSharedMemory: false, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(full); + if (!durable) throw new Error('durable slot missing'); + await waitForJob(durable); + + expect(fixture.run).toHaveBeenCalledTimes(1); + expect(full).toMatchObject({ + includeSharedMemory: true, + status: 'deferred', + }); + expect(durable).toMatchObject({ + includeSharedMemory: false, + status: 'done', + }); + expect(fixture.writeReadiness).toHaveBeenCalledTimes(1); + expect(fixture.writeReadiness).toHaveBeenCalledWith( + 'cg:scope-backpressure', + { durableVerified: true, sharedMemoryVerified: false }, + ); + }); + + it('keeps denied mixed progress deferred when local backpressure left the round incomplete', async () => { + const result = publicDurableAndSharedMemoryResult(); + result.denied = true; + result.deniedPeers = 1; + result.deferredBackpressure = 1; + const fixture = coordinatorFixture({ result }); + const job = fixture.service.start({ + contextGraphId: 'cg:denied-deferred', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + + await waitForJob(job); + + expect(job).toMatchObject({ + status: 'deferred', + error: expect.stringContaining('local scheduler backpressure'), + }); + expect(fixture.writeReadiness).not.toHaveBeenCalled(); + expect(fixture.markSubscriptionState).not.toHaveBeenCalled(); + expect(fixture.hasConfirmedMeta).toHaveBeenCalledTimes(1); + expect(fixture.isPrivate).toHaveBeenCalledTimes(1); + }); + + it('keeps orchestration state out of the historical two-map tracker', () => { + const tracker: CatchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; + const service = new ContextGraphCatchupCoordinatorService(tracker, { + runner: { run: vi.fn() }, + readReadiness: vi.fn(), + hasConfirmedMeta: vi.fn(), + isPrivate: vi.fn(), + writeReadiness: vi.fn(), + markSubscriptionState: vi.fn(), + emitProjectSynced: vi.fn(), + }); + + expect(service.coalesceActive({ + contextGraphId: 'cg:legacy-tracker', + includeSharedMemory: true, + })).toBeUndefined(); + expect(Object.keys(tracker).sort()).toEqual(['jobs', 'latestByContextGraph']); + }); + + it('refreshes latest status when broad, narrow, then broad reuses existing views', async () => { + const fixture = coordinatorFixture({ blockBroadBase: true }); + const broad = fixture.service.start({ + contextGraphId: 'cg:broad-narrow-broad', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + + const narrow = fixture.service.coalesceActive({ + contextGraphId: 'cg:broad-narrow-broad', + includeSharedMemory: false, + }); + expect(fixture.tracker.latestByContextGraph.get('cg:broad-narrow-broad')) + .toBe(narrow?.jobId); + + const reusedBroad = fixture.service.coalesceActive({ + contextGraphId: 'cg:broad-narrow-broad', + includeSharedMemory: true, + }); + expect(reusedBroad?.jobId).toBe(broad.jobId); + expect(fixture.tracker.latestByContextGraph.get('cg:broad-narrow-broad')) + .toBe(broad.jobId); + + fixture.releaseFirstRun.resolve(); + await waitForJob(broad); + if (!narrow) throw new Error('narrow view missing'); + await waitForJob(narrow); + }); + + it('refreshes latest status when narrow, upgrade, then narrow reuses the base view', async () => { + const fixture = coordinatorFixture(); + const narrow = fixture.service.start({ + contextGraphId: 'cg:narrow-broad-narrow', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + + const broad = fixture.service.coalesceActive({ + contextGraphId: 'cg:narrow-broad-narrow', + includeSharedMemory: true, + }); + expect(fixture.tracker.latestByContextGraph.get('cg:narrow-broad-narrow')) + .toBe(broad?.jobId); + + const reusedNarrow = fixture.service.coalesceActive({ + contextGraphId: 'cg:narrow-broad-narrow', + includeSharedMemory: false, + }); + expect(reusedNarrow?.jobId).toBe(narrow.jobId); + expect(fixture.tracker.latestByContextGraph.get('cg:narrow-broad-narrow')) + .toBe(narrow.jobId); + + fixture.releaseFirstRun.resolve(); + await waitForJob(narrow); + if (!broad) throw new Error('broad upgrade missing'); + await waitForJob(broad); + }); + + it('keeps job scopes immutable and serializes one wider upgrade', async () => { + const fixture = coordinatorFixture(); + const base = fixture.service.start({ + contextGraphId: 'cg:one', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:one', + includeSharedMemory: true, + }); + const repeatedUpgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:one', + includeSharedMemory: true, + }); + + expect(base).toMatchObject({ jobId: 'job-1', includeSharedMemory: false }); + expect(upgrade).toMatchObject({ jobId: 'job-2', includeSharedMemory: true }); + expect(repeatedUpgrade?.jobId).toBe(upgrade?.jobId); + expect(fixture.run).toHaveBeenCalledTimes(1); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(fixture.run.mock.calls.map(([request]) => request)).toEqual([ + { contextGraphId: 'cg:one', includeSharedMemory: false }, + { contextGraphId: 'cg:one', includeSharedMemory: true }, + ]); + expect(base).toMatchObject({ includeSharedMemory: false, status: 'done' }); + expect(upgrade).toMatchObject({ includeSharedMemory: true, status: 'done' }); + expect(fixture.service.coalesceActive({ + contextGraphId: 'cg:one', + includeSharedMemory: true, + })).toBeUndefined(); + }); + + it('does not retroactively fail VM-only success when the wider upgrade is incomplete', async () => { + const fixture = coordinatorFixture({ failUpgrade: true }); + const base = fixture.service.start({ + contextGraphId: 'cg:two', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:two', + includeSharedMemory: true, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(base).toMatchObject({ includeSharedMemory: false, status: 'done' }); + expect(upgrade).toMatchObject({ + includeSharedMemory: true, + status: 'unreachable', + error: expect.stringContaining('requested data plane'), + }); + }); + + it('settles a queued wider job when the base attempt throws', async () => { + const fixture = coordinatorFixture({ baseOutcome: 'throw' }); + const base = fixture.service.start({ + contextGraphId: 'cg:failed-base', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:failed-base', + includeSharedMemory: true, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(fixture.run).toHaveBeenCalledTimes(1); + expect(base).toMatchObject({ status: 'failed', error: 'base attempt failed' }); + expect(upgrade).toMatchObject({ + status: 'failed', + error: 'base attempt failed', + finishedAt: expect.any(Number), + }); + }); + + it('settles a queued wider job without a second run when the base is denied', async () => { + const fixture = coordinatorFixture({ baseOutcome: 'denied' }); + const base = fixture.service.start({ + contextGraphId: 'cg:denied-base', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:denied-base', + includeSharedMemory: true, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(fixture.run).toHaveBeenCalledTimes(1); + expect(base.status).toBe('denied'); + expect(upgrade).toMatchObject({ + status: 'denied', + finishedAt: expect.any(Number), + }); + expect(fixture.hasConfirmedMeta).not.toHaveBeenCalled(); + expect(fixture.isPrivate).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/test/context-graph-catchup-readiness.test.ts b/packages/cli/test/context-graph-catchup-readiness.test.ts index 8071901332..cea03f237d 100644 --- a/packages/cli/test/context-graph-catchup-readiness.test.ts +++ b/packages/cli/test/context-graph-catchup-readiness.test.ts @@ -2,8 +2,14 @@ import { describe, expect, it } from 'vitest'; import type { CatchupJobResult } from '../src/catchup-runner.js'; import { CONTEXT_GRAPH_READINESS_VERSION, - classifyContextGraphCatchupReadiness, + combineCatchupPlaneEvidence, + describeContextGraphConvergence, + planContextGraphCatchupReadiness, } from '../src/context-graph-readiness.js'; +import { + publicDurableAndSharedMemoryResult, + publicDurableWithSharedMemoryBackpressureResult, +} from './helpers/context-graph-catchup-fixtures.js'; function mixedPeerResult(verifiedDataPeers: number): CatchupJobResult { return { @@ -62,6 +68,19 @@ function mixedPeerResult(verifiedDataPeers: number): CatchupJobResult { }; } +function classifyContextGraphCatchupReadiness( + input: Parameters[0] & { + hasConfirmedMeta: boolean; + isPrivate: boolean; + }, +) { + const { hasConfirmedMeta, isPrivate, ...planInput } = input; + const plan = planContextGraphCatchupReadiness(planInput); + return plan.kind === 'settled' + ? plan.classification + : plan.finalize({ hasConfirmedMeta, isPrivate }); +} + describe('context graph catch-up readiness classification', () => { const readinessBeforeCatchup = { version: 0, @@ -70,6 +89,44 @@ describe('context graph catch-up readiness classification', () => { updatedAt: 0, }; + it('scopes shared-memory backpressure out of a durable-only view', () => { + const result = publicDurableWithSharedMemoryBackpressureResult(); + const classify = (includeSharedMemory: boolean) => + classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory, + hasConfirmedMeta: true, + isPrivate: false, + readinessBeforeCatchup, + }); + + expect(classify(false)).toMatchObject({ jobStatus: 'done' }); + expect(classify(true)).toMatchObject({ + jobStatus: 'deferred', + error: expect.stringContaining('local scheduler backpressure'), + }); + }); + + it('keeps durable-plane backpressure deferred for narrow and broad views', () => { + const result = publicDurableAndSharedMemoryResult(); + result.deferredBackpressure = 1; + if (!result.diagnostics) throw new Error('diagnostics missing'); + result.diagnostics.durable.deferredBackpressure = 1; + + for (const includeSharedMemory of [false, true]) { + expect(classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory, + hasConfirmedMeta: true, + isPrivate: false, + readinessBeforeCatchup, + })).toMatchObject({ + jobStatus: 'deferred', + error: expect.stringContaining('local scheduler backpressure'), + }); + } + }); + it('uses a clean per-peer completion even when aggregate diagnostics contain denial and timeout', () => { const classification = classifyContextGraphCatchupReadiness({ result: mixedPeerResult(1), @@ -531,3 +588,110 @@ describe('context graph catch-up readiness classification', () => { }).jobStatus).toBe('unreachable'); }); }); + +describe('selected context-graph convergence snapshot', () => { + it('combines this-run evidence without reviving stale persisted proof', () => { + const planes = combineCatchupPlaneEvidence({ + readinessBeforeCatchup: { + version: 0, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: 42, + }, + durableReadyThisRun: true, + sharedMemoryReadyThisRun: false, + includeSharedMemory: true, + hasConfirmedMeta: true, + }); + + expect(planes).toEqual({ + state: 'partial', + required: { + metadata: true, + durable: true, + sharedMemory: true, + }, + verified: { + metadata: true, + durable: true, + sharedMemory: false, + }, + missing: ['sharedMemory'], + }); + }); + + it('requires independently verified VM and requested SWM planes', () => { + const snapshot = describeContextGraphConvergence({ + readiness: { + version: 1, + durableVerified: false, + sharedMemoryVerified: true, + updatedAt: 42, + }, + includeSharedMemory: true, + hasConfirmedMeta: true, + observedAt: 100, + }); + + expect(snapshot).toEqual({ + state: 'partial', + required: { + metadata: true, + durable: true, + sharedMemory: true, + }, + verified: { + metadata: true, + durable: false, + sharedMemory: true, + }, + missing: ['durable'], + readinessUpdatedAt: 42, + observedAt: 100, + }); + }); + + it('rejects stale per-plane provenance when authoritative metadata is absent', () => { + const snapshot = describeContextGraphConvergence({ + readiness: { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: 42, + }, + includeSharedMemory: true, + hasConfirmedMeta: false, + observedAt: 100, + }); + + expect(snapshot).toMatchObject({ + state: 'pending', + verified: { + metadata: false, + durable: false, + sharedMemory: false, + }, + missing: ['metadata', 'durable', 'sharedMemory'], + }); + }); + + it('marks a VM-only request complete without requiring SWM', () => { + const snapshot = describeContextGraphConvergence({ + readiness: { + version: 1, + durableVerified: true, + sharedMemoryVerified: false, + updatedAt: 42, + }, + includeSharedMemory: false, + hasConfirmedMeta: true, + observedAt: 100, + }); + + expect(snapshot).toMatchObject({ + state: 'complete', + missing: [], + required: { sharedMemory: false }, + }); + }); +}); diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index ad47313dd5..5c6df707b1 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -1,294 +1,69 @@ -import { afterEach, describe, expect, it } from 'vitest'; -import { createServer, type Server } from 'node:http'; -import type { CatchupJobResult, CatchupRunRequest } from '../src/catchup-runner.js'; -import { handleContextGraphRoutes } from '../src/daemon/routes/context-graph.js'; -import { handleQueryRoutes } from '../src/daemon/routes/query.js'; -import { daemonState } from '../src/daemon/state.js'; - -function cleanEmptyResult(): CatchupJobResult { - return { - connectedPeers: 1, - totalPeers: 1, - selectedPeers: 1, - syncCapablePeers: 1, - peersTried: 1, - peersResponded: 1, - peersSucceeded: 1, - dataSynced: 0, - sharedMemorySynced: 0, - denied: false, - deniedPeers: 0, - cleanPlaneCompletions: { - durable: { verifiedDataPeers: 0, emptyPeers: 1 }, - sharedMemory: { verifiedDataPeers: 0, emptyPeers: 1 }, - }, - diagnostics: { - noProtocolPeers: 0, - durable: { - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 0, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 1, - metaOnlyResponses: 0, - dataRejectedMissingMeta: 0, - rejectedKcs: 0, - failedPeers: 0, - failedPhases: 0, - }, - sharedMemory: { - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 0, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 1, - droppedDataTriples: 0, - failedPeers: 0, - failedPhases: 0, - }, - }, - }; -} - -function privateMetaOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable) throw new Error('durable diagnostics missing'); - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedMetaTriples = 7; - result.diagnostics.durable.insertedMetaTriples = 1; - result.diagnostics.durable.metaOnlyResponses = 1; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.durable.emptyPeers = 0; - return result; -} - -function privateDataOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { - throw new Error('catch-up diagnostics missing'); - } - result.dataSynced = 3; - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedDataTriples = 3; - result.diagnostics.durable.insertedDataTriples = 3; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.completedPhases = 0; - result.diagnostics.sharedMemory.timedOutPhases = 1; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 0, emptyPeers: 0 }; - return result; -} - -function privateSharedMemoryOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.sharedMemory) { - throw new Error('shared-memory diagnostics missing'); - } - result.sharedMemorySynced = 4; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.fetchedDataTriples = 4; - result.diagnostics.sharedMemory.insertedDataTriples = 4; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; - return result; -} - -function publicDurableAndSharedMemoryResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { - throw new Error('catch-up diagnostics missing'); - } - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.dataSynced = 3; - result.sharedMemorySynced = 4; - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedDataTriples = 3; - result.diagnostics.durable.insertedDataTriples = 3; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.fetchedDataTriples = 4; - result.diagnostics.sharedMemory.insertedDataTriples = 4; - result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; - return result; -} +import { describe, expect, it } from 'vitest'; +import { DKGEvent } from '@origintrail-official/dkg-core'; +import { + cleanEmptyResult, + privateDataOnlyResult, + privateMetaOnlyResult, + privateSharedMemoryOnlyResult, + publicDurableAndSharedMemoryResult, +} from './helpers/context-graph-catchup-fixtures.js'; +import { runSubscribeScenario as subscribe } from './helpers/context-graph-subscribe-route-harness.js'; describe('context graph subscribe readiness requires authoritative metadata', () => { - const previousCatchupRunner = daemonState.catchupRunner; - let server: Server | undefined; + it('keeps omitted sync mode backward-compatible as always-on', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + }); + + expect(result.response.syncMode).toBe('always-on'); + expect(result.subscribeCalls).toEqual([ + { id: expect.any(String), options: { syncMode: 'always-on' } }, + ]); + expect(result.state.syncMode).toBe('always-on'); + }); - afterEach(async () => { - daemonState.catchupRunner = previousCatchupRunner; - if (!server) return; - await new Promise((resolve, reject) => { - server!.close((err) => (err ? reject(err) : resolve())); + it('forwards explicit on-demand edge intent without making it always-on', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + syncMode: 'on-demand', }); - server = undefined; + + expect(result.response.syncMode).toBe('on-demand'); + expect(result.subscribeCalls).toEqual([ + { id: expect.any(String), options: { syncMode: 'on-demand' } }, + ]); + expect(result.state.syncMode).toBe('on-demand'); }); - async function subscribe(opts: { - initial?: Record; - hasConfirmedMeta: boolean; - hasConfirmedMetaAfterCatchup?: boolean; - isPrivate?: boolean; - allowedAgents?: string[]; - callerAddress?: string; - result?: CatchupJobResult; - includeSharedMemory?: boolean; - readiness?: { - version: number; - durableVerified: boolean; - sharedMemoryVerified: boolean; - updatedAt?: number; - }; - }): Promise<{ - response: any; - job: any; - runCalls: number; - runRequests: CatchupRunRequest[]; - state: Record; - patches: Array>; - readiness: Record | undefined; - statusResponse: any; - }> { - const contextGraphId = `readiness-${Math.random().toString(36).slice(2, 8)}`; - const state = new Map>(); - if (opts.initial) state.set(contextGraphId, { ...opts.initial }); - const patches: Array> = []; - const catchupTracker = { - jobs: new Map(), - latestByContextGraph: new Map(), - }; - let runCalls = 0; - const runRequests: CatchupRunRequest[] = []; - let readiness = opts.readiness - ? { ...opts.readiness, updatedAt: opts.readiness.updatedAt ?? Date.now() } - : undefined; - - daemonState.catchupRunner = { - run: async (request) => { - runCalls += 1; - runRequests.push(request); - return opts.result ?? cleanEmptyResult(); - }, - close: async () => {}, - }; - - const agent = { - getContextGraphAllowedAgents: async () => opts.allowedAgents ?? [], - getSubscribedContextGraphs: () => state, - subscribeToContextGraph: (id: string) => { - state.set(id, { - ...state.get(id), - subscribed: true, - }); - }, - markContextGraphSubscriptionState: (id: string, patch: Record) => { - patches.push({ ...patch }); - state.set(id, { ...state.get(id), ...patch }); - }, - hasConfirmedMetaState: async () => { - return runCalls > 0 - ? opts.hasConfirmedMetaAfterCatchup ?? opts.hasConfirmedMeta - : opts.hasConfirmedMeta; + it('reports the agent-applied mode when an on-demand open cannot downgrade always-on', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + syncMode: 'on-demand', + initial: { + subscribed: true, + syncMode: 'always-on', + synced: false, }, - isPrivateContextGraph: async () => opts.isPrivate ?? false, - resolveAgentByToken: () => undefined, - getDefaultAgentAddress: () => opts.callerAddress ?? '0x0000000000000000000000000000000000000001', - }; - - server = createServer(async (req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); - const routeContext = { - req, - res, - agent, - publisherControl: {}, - publisherRuntime: null, - config: { auth: { enabled: false } }, - startedAt: Date.now(), - dashDb: { - getContextGraphReadinessProvenance: () => readiness ?? null, - setContextGraphReadinessProvenance: (_id: string, next: Record) => { - readiness = { ...next, updatedAt: Date.now() } as typeof readiness; - }, - }, - opWallets: {}, - network: {}, - tracker: {}, - memoryManager: {}, - bridgeAuthToken: undefined, - nodeVersion: 'test', - nodeCommit: 'test', - catchupTracker, - extractionRegistry: {}, - fileStore: {}, - extractionStatus: new Map(), - assertionImportLocks: new Map(), - vectorStore: {}, - embeddingProvider: null, - validTokens: new Set(), - apiHost: '127.0.0.1', - apiPortRef: { value: 0 }, - routePlugins: [], - url, - path: url.pathname, - requestToken: undefined, - requestAgentAddress: undefined, - } as any; - await handleContextGraphRoutes(routeContext); - if (!res.writableEnded) await handleQueryRoutes(routeContext); - if (!res.writableEnded) { - res.statusCode = 404; - res.end(); - } - }); - await new Promise((resolve) => server!.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (!address || typeof address === 'string') throw new Error('route test server did not bind'); - - const httpResponse = await fetch(`http://127.0.0.1:${address.port}/api/context-graph/subscribe`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - contextGraphId, - includeSharedMemory: opts.includeSharedMemory ?? true, - }), - }); - const response = await httpResponse.json() as any; - const jobId = response.catchup.jobId as string; - - for (let i = 0; i < 50; i++) { - if (catchupTracker.jobs.get(jobId)?.finishedAt) break; - await new Promise((resolve) => setTimeout(resolve, 5)); - } + }); + + expect(result.subscribeCalls).toEqual([ + { id: expect.any(String), options: { syncMode: 'on-demand' } }, + ]); + expect(result.response.syncMode).toBe('always-on'); + expect(result.state.syncMode).toBe('always-on'); + }); - const statusHttpResponse = await fetch( - `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, - ); - const statusResponse = await statusHttpResponse.json() as any; - - return { - response, - job: catchupTracker.jobs.get(jobId), - runCalls, - runRequests, - state: state.get(contextGraphId) ?? {}, - patches, - readiness, - statusResponse, - }; - } + it('rejects unknown sync modes before changing subscription state', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + syncMode: 'sometimes', + }); + + expect(result.responseStatus).toBe(400); + expect(result.response.error).toContain('Invalid "syncMode"'); + expect(result.subscribeCalls).toEqual([]); + expect(result.runCalls).toBe(0); + }); it('does not turn a clean empty response with no authoritative metadata into ready state', async () => { const result = await subscribe({ @@ -314,6 +89,9 @@ describe('context graph subscribe readiness requires authoritative metadata', () pendingMeta: true, }); expect(result.patches).not.toContainEqual(expect.objectContaining({ synced: true })); + expect(result.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: true, + }); }); it('bypasses synthetic done and heals poisoned ready flags when metaSynced is false', async () => { @@ -347,7 +125,10 @@ describe('context graph subscribe readiness requires authoritative metadata', () it('revalidates a stale metaSynced=true bit before returning synthetic done', async () => { const result = await subscribe({ - hasConfirmedMeta: false, + // The permissive legacy check sees a placeholder, but the canonical + // readiness boundary must reject it for a remotely curated graph. + hasConfirmedMeta: true, + strictHasConfirmedMeta: false, initial: { subscribed: true, synced: true, @@ -366,6 +147,9 @@ describe('context graph subscribe readiness requires authoritative metadata', () metaSynced: false, pendingMeta: true, }); + expect(result.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: true, + }); }); it('does not restore stale true provenance after metadata arrives during an unclean catch-up', async () => { @@ -509,6 +293,16 @@ describe('context graph subscribe readiness requires authoritative metadata', () expect(result.statusResponse).toMatchObject({ jobId: result.response.catchup.jobId, status: 'done', + convergence: { + state: 'complete', + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + automaticRetryActive: true, + }, result: { dataSynced: 3, sharedMemorySynced: 4, @@ -526,6 +320,15 @@ describe('context graph subscribe readiness requires authoritative metadata', () durableVerified: true, sharedMemoryVerified: true, }); + expect(result.emittedEvents).toContainEqual({ + event: DKGEvent.PROJECT_SYNCED, + payload: { + contextGraphId: result.response.subscribed, + dataSynced: 3, + sharedMemorySynced: 4, + verifiedPrivateOnlyResponses: 0, + }, + }); }); // Issue #2006: an empty response cannot distinguish "hosts an empty graph" @@ -691,15 +494,17 @@ describe('context graph subscribe readiness requires authoritative metadata', () }); expect(result.runCalls).toBe(1); - expect(result.job.status).toBe('done'); - expect(result.job.error).toBeUndefined(); + expect(result.job).toMatchObject({ + status: 'unreachable', + error: expect.stringContaining('durable VM'), + }); expect(result.job.result).toMatchObject({ dataSynced: 0, sharedMemorySynced: 4, }); expect(result.state).toMatchObject({ subscribed: true, - synced: true, + synced: false, sharedMemorySynced: true, metaSynced: true, pendingMeta: false, diff --git a/packages/cli/test/daemon-context-graph-bootstrap.test.ts b/packages/cli/test/daemon-context-graph-bootstrap.test.ts index fe9583e47d..04cff4a1e1 100644 --- a/packages/cli/test/daemon-context-graph-bootstrap.test.ts +++ b/packages/cli/test/daemon-context-graph-bootstrap.test.ts @@ -84,6 +84,7 @@ describe('configured context graph daemon bootstrap', () => { expect(fixture.ensureContextGraphLocal).not.toHaveBeenCalled(); expect(fixture.subscribeToContextGraph).toHaveBeenCalledWith( '0x1234567890123456789012345678901234567890/private-cg', + { syncMode: 'always-on' }, ); expect( fixture.subscriptions.get('0x1234567890123456789012345678901234567890/private-cg'), @@ -148,6 +149,7 @@ describe('configured context graph daemon bootstrap', () => { expect(fixture.subscribeToContextGraph).toHaveBeenCalledWith( '0x1234567890123456789012345678901234567890/local-cg', + { syncMode: 'always-on' }, ); expect(fixture.markContextGraphSubscriptionState).not.toHaveBeenCalled(); expect( @@ -434,6 +436,7 @@ describe('configured context graph daemon bootstrap', () => { expect(fixture.subscribeToContextGraph).toHaveBeenCalledTimes(1); expect(fixture.subscribeToContextGraph).toHaveBeenCalledWith( '0x1234567890123456789012345678901234567890/remote-cg', + { syncMode: 'always-on' }, ); expect(fixture.subscriptions.has(SYSTEM_CONTEXT_GRAPHS.AGENTS)).toBe(false); expect(fixture.subscriptions.has(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY)).toBe(false); diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index 951b3925e0..5eb50800b5 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -57,6 +57,7 @@ import { getSharedContext, HARDHAT_KEYS } from '../../chain/test/evm-test-contex import { ApiClient } from '../src/api-client.js'; import { handleContextGraphRoutes } from '../src/daemon/routes/context-graph.js'; import { daemonState } from '../src/daemon/state.js'; +import { createContextGraphCatchupRouteAdapter } from '../src/daemon/context-graph-catchup-route-adapter.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const CLI_ENTRY = join(__dirname, '..', 'dist', 'cli.js'); @@ -994,7 +995,10 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { it('marks timeout-after-response catchup as failed rather than unreachable', async () => { const contextGraphId = 'catchup-timeout-response-' + Math.random().toString(36).slice(2, 8); - const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map() }; + const catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { run: async () => ({ @@ -1051,7 +1055,11 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { const agent = { getContextGraphAllowedAgents: async () => [], getSubscribedContextGraphs: () => new Map(), - subscribeToContextGraph: () => {}, + subscribeToContextGraph: () => ({ + subscribed: true, + synced: false, + syncMode: 'always-on' as const, + }), contextGraphHasLocalContent: async () => false, markContextGraphSubscriptionState: () => { throw new Error('timeout-only catchup must not mark subscription synced'); @@ -1059,6 +1067,12 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { resolveAgentByToken: () => undefined, getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, res, @@ -1076,6 +1090,7 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), @@ -1140,7 +1155,10 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { it('marks local scheduler deferral retryable without setting shared-memory completion', async () => { const contextGraphId = 'catchup-local-deferral-' + Math.random().toString(36).slice(2, 8); - const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map() }; + const catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { run: async () => ({ @@ -1166,12 +1184,22 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { const agent = { getContextGraphAllowedAgents: async () => [], getSubscribedContextGraphs: () => new Map(), - subscribeToContextGraph: () => {}, + subscribeToContextGraph: () => ({ + subscribed: true, + synced: false, + syncMode: 'always-on' as const, + }), contextGraphHasLocalContent: async () => true, markContextGraphSubscriptionState: () => { markedSynced = true; }, resolveAgentByToken: () => undefined, getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, res, @@ -1189,6 +1217,7 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), @@ -1239,6 +1268,143 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { } }); + it('applies syncMode through the legacy /api/subscribe alias and reports the agent-normalized mode', async () => { + const contextGraphId = 'legacy-subscribe-mode-' + Math.random().toString(36).slice(2, 8); + const catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; + let requestedMode: string | undefined; + let runnerCalls = 0; + const previousCatchupRunner = daemonState.catchupRunner; + daemonState.catchupRunner = { + run: async () => { + runnerCalls += 1; + return { + connectedPeers: 0, + syncCapablePeers: 0, + peersTried: 0, + peersResponded: 0, + peersSucceeded: 0, + deferredBackpressure: 1, + dataSynced: 0, + sharedMemorySynced: 0, + denied: false, + deniedPeers: 0, + }; + }, + close: async () => {}, + } as any; + let routeServer: Server | null = null; + + try { + routeServer = createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + const agent = { + getContextGraphAllowedAgents: async () => [], + getSubscribedContextGraphs: () => new Map(), + subscribeToContextGraph: (_id: string, opts: { syncMode: string }) => { + requestedMode = opts.syncMode; + return { + subscribed: true, + synced: false, + // The route must report the normalized mode returned by the + // agent, rather than independently reconstructing it. + syncMode: 'always-on' as const, + }; + }, + markContextGraphSubscriptionState: () => {}, + resolveAgentByToken: () => undefined, + getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', + }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); + + await handleContextGraphRoutes({ + req, + res, + agent, + publisherControl: {}, + publisherRuntime: null, + config: {}, + startedAt: Date.now(), + dashDb: {}, + opWallets: {}, + network: {}, + tracker: {}, + memoryManager: {}, + bridgeAuthToken: undefined, + nodeVersion: 'test', + nodeCommit: 'test', + catchupTracker, + catchupCoordinator, + extractionRegistry: {}, + fileStore: {}, + extractionStatus: new Map(), + assertionImportLocks: new Map(), + vectorStore: {}, + embeddingProvider: null, + validTokens: new Set(), + apiHost: '127.0.0.1', + apiPortRef: { value: 0 }, + routePlugins: [], + url, + path: url.pathname, + requestToken: undefined, + requestAgentAddress: '0x0000000000000000000000000000000000000001', + } as any); + if (!res.writableEnded) { + res.statusCode = 404; + res.end(); + } + }); + + await new Promise((resolve) => routeServer!.listen(0, '127.0.0.1', resolve)); + const address = routeServer.address(); + if (!address || typeof address === 'string') { + throw new Error('legacy subscribe route test server did not bind to a TCP port'); + } + + const invalidResponse = await fetch(`http://127.0.0.1:${address.port}/api/subscribe`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ contextGraphId, syncMode: null }), + }); + expect(invalidResponse.status).toBe(400); + expect(await invalidResponse.json()).toMatchObject({ + error: 'Invalid "syncMode" (expected "on-demand" or "always-on")', + }); + expect(requestedMode).toBeUndefined(); + expect(runnerCalls).toBe(0); + expect(catchupTracker.jobs.size).toBe(0); + expect(catchupTracker.latestByContextGraph.size).toBe(0); + + const response = await fetch(`http://127.0.0.1:${address.port}/api/subscribe`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ contextGraphId, syncMode: 'on-demand' }), + }); + expect(response.status).toBe(200); + expect(requestedMode).toBe('on-demand'); + expect(await response.json()).toMatchObject({ + subscribed: contextGraphId, + syncMode: 'always-on', + catchup: { status: 'queued' }, + }); + } finally { + daemonState.catchupRunner = previousCatchupRunner; + if (routeServer) { + await new Promise((resolve, reject) => { + routeServer!.close((err) => (err ? reject(err) : resolve())); + }); + } + } + }); + // SPEC_CG_MEMORY_MODEL / Codex PR #595 round-4: per-CG hosting // committees and per-CG quorum overrides were removed end-to-end. // The on-chain contract no longer accepts those args, so silently @@ -2300,11 +2466,22 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli getSubscribedContextGraphs: () => new Map(), subscribeToContextGraph: () => { subscribeCalled = true; + return { + subscribed: true, + synced: false, + syncMode: 'always-on' as const, + }; }, contextGraphHasLocalContent: async () => false, markContextGraphSubscriptionState: () => {}, resolveAgentByToken: () => undefined, }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, res, @@ -2322,6 +2499,7 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), diff --git a/packages/cli/test/helpers/context-graph-catchup-fixtures.ts b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts new file mode 100644 index 0000000000..c686854e93 --- /dev/null +++ b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts @@ -0,0 +1,241 @@ +import type { CatchupJobResult } from '../../src/catchup-runner.js'; + +type CleanPlaneCompletions = NonNullable< + CatchupJobResult['cleanPlaneCompletions'] +>; +type CatchupDiagnostics = NonNullable; + +export type CatchupJobResultOverrides = Omit< + Partial, + 'cleanPlaneCompletions' | 'diagnostics' +> & { + cleanPlaneCompletions?: { + durable?: Partial; + sharedMemory?: Partial; + }; + diagnostics?: { + noProtocolPeers?: number; + durable?: Partial; + sharedMemory?: Partial; + }; +}; + +/** Complete canonical result factory; scenario helpers override only their signal. */ +export function makeCatchupJobResult( + overrides: CatchupJobResultOverrides = {}, +): CatchupJobResult { + const defaults = { + connectedPeers: 1, + totalPeers: 1, + selectedPeers: 1, + syncCapablePeers: 1, + peersTried: 1, + peersResponded: 1, + peersSucceeded: 1, + deferredBackpressure: 0, + dataSynced: 0, + sharedMemorySynced: 0, + denied: false, + deniedPeers: 0, + cleanPlaneCompletions: { + durable: { + verifiedDataPeers: 0, + verifiedPrivateOnlyPeers: 0, + emptyPeers: 1, + }, + sharedMemory: { + verifiedDataPeers: 0, + emptyPeers: 1, + }, + }, + diagnostics: { + noProtocolPeers: 0, + durable: { + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 0, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 1, + metaOnlyResponses: 0, + verifiedPrivateOnlyResponses: 0, + dataRejectedMissingMeta: 0, + rejectedKcs: 0, + failedPeers: 0, + failedPhases: 0, + deferredBackpressure: 0, + }, + sharedMemory: { + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 0, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 1, + droppedDataTriples: 0, + failedPeers: 0, + failedPhases: 0, + deferredBackpressure: 0, + }, + }, + } satisfies CatchupJobResult; + + return { + ...defaults, + ...overrides, + cleanPlaneCompletions: { + durable: { + ...defaults.cleanPlaneCompletions.durable, + ...overrides.cleanPlaneCompletions?.durable, + }, + sharedMemory: { + ...defaults.cleanPlaneCompletions.sharedMemory, + ...overrides.cleanPlaneCompletions?.sharedMemory, + }, + }, + diagnostics: { + noProtocolPeers: + overrides.diagnostics?.noProtocolPeers ?? defaults.diagnostics.noProtocolPeers, + durable: { + ...defaults.diagnostics.durable, + ...overrides.diagnostics?.durable, + }, + sharedMemory: { + ...defaults.diagnostics.sharedMemory, + ...overrides.diagnostics?.sharedMemory, + }, + }, + }; +} + +export function cleanEmptyResult(): CatchupJobResult { + return makeCatchupJobResult(); +} + +export function privateMetaOnlyResult(): CatchupJobResult { + return makeCatchupJobResult({ + cleanPlaneCompletions: { + durable: { emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedMetaTriples: 7, + insertedMetaTriples: 1, + metaOnlyResponses: 1, + }, + }, + }); +} + +export function privateDataOnlyResult(): CatchupJobResult { + return makeCatchupJobResult({ + dataSynced: 3, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 1, emptyPeers: 0 }, + sharedMemory: { verifiedDataPeers: 0, emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedDataTriples: 3, + insertedDataTriples: 3, + }, + sharedMemory: { + emptyResponses: 0, + completedPhases: 0, + timedOutPhases: 1, + }, + }, + }); +} + +export function privateSharedMemoryOnlyResult(): CatchupJobResult { + return makeCatchupJobResult({ + sharedMemorySynced: 4, + cleanPlaneCompletions: { + sharedMemory: { verifiedDataPeers: 1, emptyPeers: 0 }, + }, + diagnostics: { + sharedMemory: { + emptyResponses: 0, + fetchedDataTriples: 4, + insertedDataTriples: 4, + }, + }, + }); +} + +export function publicDurableAndSharedMemoryResult(): CatchupJobResult { + return makeCatchupJobResult({ + dataSynced: 3, + sharedMemorySynced: 4, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 1, emptyPeers: 0 }, + sharedMemory: { verifiedDataPeers: 1, emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedDataTriples: 3, + insertedDataTriples: 3, + }, + sharedMemory: { + emptyResponses: 0, + fetchedDataTriples: 4, + insertedDataTriples: 4, + }, + }, + }); +} + +/** Durable VM completed, but the shared-memory scheduler deferred its plane. */ +export function publicDurableWithSharedMemoryBackpressureResult(): CatchupJobResult { + return makeCatchupJobResult({ + deferredBackpressure: 1, + dataSynced: 3, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 1, emptyPeers: 0 }, + sharedMemory: { verifiedDataPeers: 0, emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedDataTriples: 3, + insertedDataTriples: 3, + }, + sharedMemory: { + emptyResponses: 0, + completedPhases: 0, + deferredBackpressure: 1, + }, + }, + }); +} + +/** One public host proves empty while another selected peer transport-fails. */ +export function lossyPublicEmptyResult(): CatchupJobResult { + return makeCatchupJobResult({ + connectedPeers: 2, + totalPeers: 2, + selectedPeers: 2, + syncCapablePeers: 2, + peersTried: 2, + peersResponded: 1, + peersSucceeded: 1, + cleanPlaneCompletions: { + durable: { emptyPeers: 1 }, + }, + diagnostics: { + durable: { failedPeers: 1 }, + }, + }); +} diff --git a/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts new file mode 100644 index 0000000000..3ffd5c21f3 --- /dev/null +++ b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts @@ -0,0 +1,358 @@ +import { createServer, type Server } from 'node:http'; +import { DKGEvent } from '@origintrail-official/dkg-core'; +import type { + CatchupJobResult, + CatchupRunRequest, +} from '../../src/catchup-runner.js'; +import { handleContextGraphRoutes } from '../../src/daemon/routes/context-graph.js'; +import { handleQueryRoutes } from '../../src/daemon/routes/query.js'; +import { daemonState } from '../../src/daemon/state.js'; +import { createContextGraphCatchupRouteAdapter } from '../../src/daemon/context-graph-catchup-route-adapter.js'; +import type { CatchupJob } from '../../src/daemon/types.js'; +import { cleanEmptyResult } from './context-graph-catchup-fixtures.js'; + +type SyncMode = 'on-demand' | 'always-on'; + +export interface SubscribeRouteHarnessOptions { + initial?: Record; + hasConfirmedMeta: boolean; + hasConfirmedMetaAfterCatchup?: boolean; + strictHasConfirmedMeta?: boolean; + locallyCurated?: boolean; + isPrivate?: boolean; + allowedAgents?: string[]; + callerAddress?: string; + readiness?: { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + updatedAt?: number; + }; + runner?: ( + request: CatchupRunRequest, + callNumber: number, + ) => Promise | CatchupJobResult; +} + +export class ContextGraphSubscribeRouteHarness { + readonly contextGraphId = `readiness-${Math.random().toString(36).slice(2, 8)}`; + readonly state = new Map>(); + readonly patches: Array> = []; + readonly runRequests: CatchupRunRequest[] = []; + readonly subscribeCalls: Array<{ + id: string; + options: { syncMode?: SyncMode } | undefined; + }> = []; + readonly metadataCheckOptions: Array< + { rejectUnregisteredPlaceholder?: boolean } | undefined + > = []; + readonly emittedEvents: Array<{ + event: DKGEvent; + payload: unknown; + }> = []; + + private readonly previousCatchupRunner = daemonState.catchupRunner; + private readonly catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; + private server: Server | undefined; + private addressPort = 0; + private runCallsValue = 0; + private readinessValue: + | { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + updatedAt: number; + } + | undefined; + + private constructor(private readonly options: SubscribeRouteHarnessOptions) { + if (options.initial) { + this.state.set(this.contextGraphId, { ...options.initial }); + } + this.readinessValue = options.readiness + ? { + ...options.readiness, + updatedAt: options.readiness.updatedAt ?? Date.now(), + } + : undefined; + } + + static async create( + options: SubscribeRouteHarnessOptions, + ): Promise { + const harness = new ContextGraphSubscribeRouteHarness(options); + await harness.start(); + return harness; + } + + get runCalls(): number { + return this.runCallsValue; + } + + get readiness(): Record | undefined { + return this.readinessValue; + } + + async postSubscribe(input?: { + includeSharedMemory?: boolean; + syncMode?: unknown; + }): Promise<{ status: number; body: any }> { + const response = await fetch( + `http://127.0.0.1:${this.addressPort}/api/context-graph/subscribe`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + contextGraphId: this.contextGraphId, + includeSharedMemory: input?.includeSharedMemory ?? true, + ...(input?.syncMode !== undefined ? { syncMode: input.syncMode } : {}), + }), + }, + ); + return { status: response.status, body: await response.json() }; + } + + async waitForJob(jobId: string | undefined): Promise { + if (!jobId) return undefined; + for (let i = 0; i < 50; i += 1) { + const job = this.catchupTracker.jobs.get(jobId); + if (job?.finishedAt) return job; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + return this.catchupTracker.jobs.get(jobId); + } + + getJob(jobId: string | undefined): CatchupJob | undefined { + return jobId ? this.catchupTracker.jobs.get(jobId) : undefined; + } + + async getStatus(jobId: string): Promise { + return fetch( + `http://127.0.0.1:${this.addressPort}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, + ).then((response) => response.json()); + } + + async getStatusByContextGraph(): Promise { + return fetch( + `http://127.0.0.1:${this.addressPort}/api/sync/catchup-status?contextGraphId=${encodeURIComponent(this.contextGraphId)}`, + ).then((response) => response.json()); + } + + setCompleteReadiness(updatedAt = Date.now()): void { + this.readinessValue = { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt, + }; + this.state.set(this.contextGraphId, { + ...this.state.get(this.contextGraphId), + synced: true, + sharedMemorySynced: true, + metaSynced: true, + pendingMeta: false, + }); + } + + async close(): Promise { + daemonState.catchupRunner = this.previousCatchupRunner; + if (!this.server) return; + const server = this.server; + this.server = undefined; + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + + private async start(): Promise { + daemonState.catchupRunner = { + run: async (request) => { + this.runCallsValue += 1; + this.runRequests.push(request); + return this.options.runner?.(request, this.runCallsValue) ?? cleanEmptyResult(); + }, + close: async () => {}, + }; + + const agent = { + getContextGraphAllowedAgents: async () => this.options.allowedAgents ?? [], + getSubscribedContextGraphs: () => this.state, + subscribeToContextGraph: ( + id: string, + options?: { syncMode?: SyncMode }, + ) => { + this.subscribeCalls.push({ id, options }); + const previous = this.state.get(id); + const effectiveSyncMode = previous?.subscribed && previous.syncMode === 'always-on' + ? 'always-on' + : options?.syncMode ?? previous?.syncMode ?? 'always-on'; + const applied = { + ...previous, + subscribed: true, + synced: previous?.synced ?? false, + syncMode: effectiveSyncMode, + }; + this.state.set(id, applied); + return applied; + }, + markContextGraphSubscriptionState: ( + id: string, + patch: Record, + ) => { + this.patches.push({ ...patch }); + this.state.set(id, { ...this.state.get(id), ...patch }); + }, + hasConfirmedMetaState: async ( + _id: string, + options?: { rejectUnregisteredPlaceholder?: boolean }, + ) => { + this.metadataCheckOptions.push(options); + if ( + options?.rejectUnregisteredPlaceholder === true && + this.options.strictHasConfirmedMeta !== undefined + ) { + return this.options.strictHasConfirmedMeta; + } + return this.runCallsValue > 0 + ? this.options.hasConfirmedMetaAfterCatchup ?? this.options.hasConfirmedMeta + : this.options.hasConfirmedMeta; + }, + isCuratorOf: async () => this.options.locallyCurated ?? false, + isPrivateContextGraph: async () => this.options.isPrivate ?? false, + eventBus: { + emit: (event: DKGEvent, payload: unknown) => { + this.emittedEvents.push({ event, payload }); + }, + }, + resolveAgentByToken: () => undefined, + getDefaultAgentAddress: () => + this.options.callerAddress ?? '0x0000000000000000000000000000000000000001', + }; + + const readinessStore = { + getContextGraphReadinessProvenance: () => this.readinessValue ?? null, + setContextGraphReadinessProvenance: ( + _id: string, + next: { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + }, + ) => { + this.readinessValue = { ...next, updatedAt: Date.now() }; + }, + }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: this.catchupTracker, + runner: daemonState.catchupRunner, + readinessStore: readinessStore as any, + agent: agent as any, + }); + + this.server = createServer(async (request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + const routeContext = { + req: request, + res: response, + agent, + publisherControl: {}, + publisherRuntime: null, + config: { auth: { enabled: false } }, + startedAt: Date.now(), + dashDb: readinessStore, + opWallets: {}, + network: {}, + tracker: {}, + memoryManager: {}, + bridgeAuthToken: undefined, + nodeVersion: 'test', + nodeCommit: 'test', + catchupTracker: this.catchupTracker, + catchupCoordinator, + extractionRegistry: {}, + fileStore: {}, + extractionStatus: new Map(), + assertionImportLocks: new Map(), + vectorStore: {}, + embeddingProvider: null, + validTokens: new Set(), + apiHost: '127.0.0.1', + apiPortRef: { value: 0 }, + routePlugins: [], + url, + path: url.pathname, + requestToken: undefined, + requestAgentAddress: undefined, + } as any; + await handleContextGraphRoutes(routeContext); + if (!response.writableEnded) await handleQueryRoutes(routeContext); + if (!response.writableEnded) { + response.statusCode = 404; + response.end(); + } + }); + await new Promise((resolve) => + this.server!.listen(0, '127.0.0.1', resolve)); + const address = this.server.address(); + if (!address || typeof address === 'string') { + throw new Error('route test server did not bind'); + } + this.addressPort = address.port; + } +} + +export interface RunSubscribeScenarioOptions + extends Omit { + result?: CatchupJobResult; + includeSharedMemory?: boolean; + syncMode?: unknown; +} + +export async function runSubscribeScenario( + options: RunSubscribeScenarioOptions, +): Promise<{ + response: any; + responseStatus: number; + job: CatchupJob | undefined; + runCalls: number; + runRequests: CatchupRunRequest[]; + subscribeCalls: ContextGraphSubscribeRouteHarness['subscribeCalls']; + state: Record; + patches: Array>; + readiness: Record | undefined; + statusResponse: any; + metadataCheckOptions: ContextGraphSubscribeRouteHarness['metadataCheckOptions']; + emittedEvents: ContextGraphSubscribeRouteHarness['emittedEvents']; +}> { + const harness = await ContextGraphSubscribeRouteHarness.create({ + ...options, + runner: () => options.result ?? cleanEmptyResult(), + }); + try { + const posted = await harness.postSubscribe({ + includeSharedMemory: options.includeSharedMemory, + syncMode: options.syncMode, + }); + const jobId = posted.body.catchup?.jobId as string | undefined; + const job = await harness.waitForJob(jobId); + return { + response: posted.body, + responseStatus: posted.status, + job, + runCalls: harness.runCalls, + runRequests: [...harness.runRequests], + subscribeCalls: [...harness.subscribeCalls], + state: harness.state.get(harness.contextGraphId) ?? {}, + patches: [...harness.patches], + readiness: harness.readiness, + statusResponse: jobId ? await harness.getStatus(jobId) : null, + metadataCheckOptions: [...harness.metadataCheckOptions], + emittedEvents: [...harness.emittedEvents], + }; + } finally { + await harness.close(); + } +} diff --git a/packages/cli/test/knowledge-subscribe-command.test.ts b/packages/cli/test/knowledge-subscribe-command.test.ts new file mode 100644 index 0000000000..52c865ad84 --- /dev/null +++ b/packages/cli/test/knowledge-subscribe-command.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { Command } from 'commander'; + +const configMocks = vi.hoisted(() => ({ + loadConfig: vi.fn(async () => ({ contextGraphs: [] as string[] })), + saveConfig: vi.fn(async () => undefined), + resolveContextGraphs: vi.fn((config: { contextGraphs?: string[] }) => config.contextGraphs ?? []), +})); + +vi.mock('../src/config.js', async (importOriginal) => ({ + ...await importOriginal(), + loadConfig: configMocks.loadConfig, + saveConfig: configMocks.saveConfig, + resolveContextGraphs: configMocks.resolveContextGraphs, +})); + +import { ApiClient } from '../src/api-client.js'; +import { registerKnowledgeCommands } from '../src/commands/knowledge.js'; + +function commandProgram(): Command { + const program = new Command().name('dkg'); + program.exitOverride(); + registerKnowledgeCommands(program); + return program; +} + +describe('knowledge subscribe CLI sync lifetime', () => { + const logLines: string[] = []; + + beforeEach(() => { + logLines.length = 0; + configMocks.loadConfig.mockClear(); + configMocks.saveConfig.mockClear(); + configMocks.resolveContextGraphs.mockClear(); + vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + logLines.push(args.map(String).join(' ')); + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('requests process-local on-demand synchronization by default', async () => { + const subscribeToContextGraph = vi.fn().mockResolvedValue({ + subscribed: 'selected-cg', + syncMode: 'on-demand', + }); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ subscribeToContextGraph } as unknown as ApiClient); + + await commandProgram().parseAsync(['node', 'dkg', 'subscribe', 'selected-cg']); + + expect(subscribeToContextGraph).toHaveBeenCalledWith('selected-cg', { + syncMode: 'on-demand', + }); + expect(configMocks.saveConfig).not.toHaveBeenCalled(); + expect(logLines.join('\n')).toContain('Synchronization mode: on demand'); + }); + + it('requests restart-durable synchronization with --save', async () => { + const subscribeToContextGraph = vi.fn().mockResolvedValue({ + subscribed: 'selected-cg', + syncMode: 'always-on', + }); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ subscribeToContextGraph } as unknown as ApiClient); + + await commandProgram().parseAsync(['node', 'dkg', 'subscribe', 'selected-cg', '--save']); + + expect(subscribeToContextGraph).toHaveBeenCalledWith('selected-cg', { + syncMode: 'always-on', + }); + expect(configMocks.saveConfig).toHaveBeenCalledWith(expect.objectContaining({ + contextGraphs: ['selected-cg'], + })); + expect(logLines.join('\n')).toContain('Synchronization mode: always on'); + }); + + it('reports the server-normalized mode when an on-demand request stays always-on', async () => { + const subscribeToContextGraph = vi.fn().mockResolvedValue({ + subscribed: 'selected-cg', + syncMode: 'always-on', + }); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ subscribeToContextGraph } as unknown as ApiClient); + + await commandProgram().parseAsync(['node', 'dkg', 'subscribe', 'selected-cg']); + + expect(subscribeToContextGraph).toHaveBeenCalledWith('selected-cg', { + syncMode: 'on-demand', + }); + expect(configMocks.saveConfig).not.toHaveBeenCalled(); + expect(logLines.join('\n')).toContain('Synchronization mode: always on'); + expect(logLines.join('\n')).not.toContain('Synchronization mode: on demand'); + }); +}); diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index 04464f9d5e..79a6ded336 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -47,7 +47,12 @@ export default defineConfig({ // Private-CG bootstrap readiness: clean-empty responses are only // terminal when authoritative metadata has been confirmed. 'test/context-graph-subscribe-readiness.test.ts', + 'test/context-graph-catchup-coalescing-route.test.ts', + 'test/context-graph-catchup-coordinator.test.ts', 'test/context-graph-catchup-readiness.test.ts', + 'test/catchup-status-response.test.ts', + 'test/catchup-status-convergence-route.test.ts', + 'test/catchup-status-cli.test.ts', 'test/context-graph-readiness-migration.test.ts', // R9 — PCA advisory wire derivation (pure) + CLI register-agent output // rendering (in-process, mocked ApiClient). No hardhat/daemon. diff --git a/packages/node-ui/src/ui/api.ts b/packages/node-ui/src/ui/api.ts index f2057bf6be..520c60479d 100644 --- a/packages/node-ui/src/ui/api.ts +++ b/packages/node-ui/src/ui/api.ts @@ -2861,8 +2861,18 @@ export const shutdownNode = () => post<{ ok: boolean }>('/api/shutdown', {}); // --- Integrations --- -export const subscribeToContextGraph = (contextGraphId: string) => - post<{ subscribed: string; catchup?: { status: string; jobId: string } }>('/api/subscribe', { contextGraphId }); +export const subscribeToContextGraph = ( + contextGraphId: string, + options?: { syncMode?: 'on-demand' | 'always-on' }, +) => + post<{ + subscribed: string; + syncMode: 'on-demand' | 'always-on'; + catchup?: { status: string; jobId: string }; + }>('/api/subscribe', { + contextGraphId, + syncMode: options?.syncMode ?? 'on-demand', + }); // --- Notifications (scoped pane wire contract — implementation-plan §3) --- // diff --git a/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx b/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx index 33fd71c301..bf6e8c88d2 100644 --- a/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx +++ b/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx @@ -325,6 +325,7 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP const [phase, setPhase] = useState('idle'); const [error, setError] = useState(null); const [pendingCgId, setPendingCgId] = useState(null); + const [keepSynced, setKeepSynced] = useState(false); // Phase 8: after approval we transition into a wire-workspace step so // the joiner can populate a local Cursor workspace from the project's // manifest. `wiredCgId` flips the modal into the WireWorkspacePanel; @@ -345,6 +346,7 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP setError(null); setPhase('idle'); setPendingCgId(null); + setKeepSynced(false); } }, [open, initialContextGraphId]); @@ -436,7 +438,9 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP // connection before the daemon snapshots its preferred/fallback peer // cohort for the catch-up job. Dial failure remains best-effort. await warmInviteCuratorConnection(intent.invite); - await subscribeToContextGraph(cgId); + await subscribeToContextGraph(cgId, { + syncMode: keepSynced ? 'always-on' : 'on-demand', + }); await refreshAndOpenContextGraph(cgId, existing); onClose(); } catch (err: unknown) { @@ -636,6 +640,27 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP spellCheck={false} /> + + {intent.kind === 'publicSubscribe' && ( + + )}
diff --git a/packages/node-ui/test/join-project-modal.interaction.test.ts b/packages/node-ui/test/join-project-modal.interaction.test.ts index 3a61ef0e74..6207724147 100644 --- a/packages/node-ui/test/join-project-modal.interaction.test.ts +++ b/packages/node-ui/test/join-project-modal.interaction.test.ts @@ -83,6 +83,7 @@ describe('JoinProjectModal public subscription interaction', () => { subscribeToContextGraphMock.mockResolvedValue({ subscribed: 'open-project', + syncMode: 'on-demand', catchup: { status: 'queued', jobId: 'catchup-1' }, }); fetchContextGraphsMock.mockResolvedValue({ @@ -148,7 +149,9 @@ describe('JoinProjectModal public subscription interaction', () => { await act(async () => { primary.click(); }); await flush(); - expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project'); + expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project', { + syncMode: 'on-demand', + }); expect(fetchContextGraphsMock).toHaveBeenCalled(); expect(signJoinRequestMock).not.toHaveBeenCalled(); expect(submitJoinRequestMock).not.toHaveBeenCalled(); @@ -167,11 +170,30 @@ describe('JoinProjectModal public subscription interaction', () => { await flush(); expect(connectToPeerIdWithTimeoutMock).toHaveBeenCalledWith(curatorPeerId); - expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project'); + expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project', { + syncMode: 'on-demand', + }); expect(connectToPeerIdWithTimeoutMock.mock.invocationCallOrder[0]).toBeLessThan( subscribeToContextGraphMock.mock.invocationCallOrder[0], ); expect(signJoinRequestMock).not.toHaveBeenCalled(); expect(submitJoinRequestMock).not.toHaveBeenCalled(); }); + + it('makes restart-durable synchronization an explicit public-graph choice', async () => { + const { container } = await renderModal('open-project'); + const keepSynced = container.querySelector( + 'input[aria-label="Keep this Context Graph synchronized after restart"]', + ) as HTMLInputElement; + expect(keepSynced.checked).toBe(false); + + await act(async () => { keepSynced.click(); }); + const primary = container.querySelector('.v10-modal-btn.primary') as HTMLButtonElement; + await act(async () => { primary.click(); }); + await flush(); + + expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project', { + syncMode: 'always-on', + }); + }); }); diff --git a/packages/node-ui/test/ui-api-pure.test.ts b/packages/node-ui/test/ui-api-pure.test.ts index 0a00cf73a1..237a62a8df 100644 --- a/packages/node-ui/test/ui-api-pure.test.ts +++ b/packages/node-ui/test/ui-api-pure.test.ts @@ -750,6 +750,14 @@ describe('UI API tests', () => { const call = requestLog.find(r => r.method === 'POST' && r.url.includes('/api/subscribe')); const body = JSON.parse(call?.body ?? '{}'); expect(body.contextGraphId).toBe('cg-1'); + expect(body.syncMode).toBe('on-demand'); + }); + + it('subscribeToContextGraph forwards an explicit always-on choice', async () => { + await subscribeToContextGraph('cg-1', { syncMode: 'always-on' }); + const call = requestLog.find(r => r.method === 'POST' && r.url.includes('/api/subscribe')); + const body = JSON.parse(call?.body ?? '{}'); + expect(body).toEqual({ contextGraphId: 'cg-1', syncMode: 'always-on' }); }); });