From ef93cd9e9a318de388fdb55652ec679ab30face2 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 02:50:53 +0200 Subject: [PATCH 01/23] feat(sync): make edge CG subscriptions on-demand by default --- packages/agent/src/dkg-agent-lifecycle.ts | 25 +++++-- packages/agent/src/dkg-agent-swm-substrate.ts | 40 ++++++++--- packages/agent/src/dkg-agent-types.ts | 12 ++++ packages/agent/src/dkg-agent.ts | 2 + packages/agent/src/index.ts | 1 + packages/agent/test/agent.part-15.test.ts | 70 ++++++++++++++++++ packages/cli/src/api-client.ts | 14 +++- packages/cli/src/commands/knowledge.ts | 9 ++- .../cli/src/daemon/routes/context-graph.ts | 21 +++++- packages/cli/test/api-client.test.ts | 21 ++++++ .../context-graph-subscribe-readiness.test.ts | 72 ++++++++++++++++--- packages/node-ui/src/ui/api.ts | 14 +++- .../ui/components/Modals/JoinProjectModal.tsx | 27 ++++++- .../join-project-modal.interaction.test.ts | 26 ++++++- packages/node-ui/test/ui-api-pure.test.ts | 8 +++ 15 files changed, 332 insertions(+), 30 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 51f29f893..3642eef5a 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -6512,9 +6512,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { ? this.contextGraphWireId(next.onChainHash) : undefined; const nextWireId = nextOnChainHash ?? localWireId; - const canonicalNext = next.onChainHash === nextOnChainHash - ? next - : { ...next, onChainHash: nextOnChainHash }; + const inheritedSyncMode = next.syncMode ?? previous?.syncMode; + const canonicalNext = { + ...next, + ...(inheritedSyncMode ? { syncMode: inheritedSyncMode } : {}), + ...(next.onChainHash === nextOnChainHash ? {} : { onChainHash: nextOnChainHash }), + }; if ( previousWireId !== nextWireId && this.wireIdToLocalCgId.get(previousWireId) === contextGraphId @@ -6526,7 +6529,11 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (!canonicalNext.subscribed && !canonicalNext.coreHosted) { this.clearVmReconcileStateForContextGraph(contextGraphId); } - if (options?.persist !== false) { + // On-demand subscriptions deliberately keep their live state and + // readiness process-local. This guard also covers later state patches + // (for example catch-up completion), so they cannot accidentally create a + // durable row after the initial subscribe call opted out. + if (options?.persist !== false && canonicalNext.syncMode !== 'on-demand') { if (this.config.contextGraphSubscriptionStore) { const revision = this.nextContextGraphSubscriptionPersistRevision(contextGraphId); this.persistContextGraphSubscription( @@ -6762,6 +6769,13 @@ export class LifecycleSyncMethods extends DKGAgentBase { return; } const sub = this.subscribedContextGraphs.get(contextGraphId); + if (sub?.syncMode === 'on-demand') { + // Some lifecycle paths persist reconciliation watermarks directly + // instead of going through setContextGraphSubscription. Preserve the + // process-local lifetime at this lowest shared write boundary too. + this.clearContextGraphSubscriptionPersistRevisionStateIfIdle(contextGraphId); + return; + } // Persist member subscriptions AND (Phase D) public CGs this Core hosts — // the host-only record MUST survive restart so a Core that was offline // during a publish remembers it hosts the CG and fills its gap. Drop the @@ -7258,6 +7272,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { const restorePendingMeta = hasJoinApproval && !approvedAgentAuthorized; this.setContextGraphSubscription(row.id, { name: row.name, + // Every row in the durable store predates or represents explicit + // restart persistence, so absence of a mode is always-on. + syncMode: 'always-on', subscribed: row.subscribed, synced: restorePendingMeta ? false : row.synced, sharedMemorySynced: restorePendingMeta ? false : row.sharedMemorySynced, diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 4893d2384..f1549c48b 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -384,11 +384,27 @@ import { DKGAgentBase } from './dkg-agent-base.js'; import type { DKGAgent } from './dkg-agent.js'; export class SwmSubstrateMethods extends DKGAgentBase { - subscribeToContextGraph(this: DKGAgent, contextGraphId: string, options?: { trackSyncScope?: boolean; persist?: boolean; deferSharedMemoryGossipSubscribe?: boolean }): void { + subscribeToContextGraph(this: DKGAgent, contextGraphId: string, options?: { + trackSyncScope?: boolean; + persist?: boolean; + deferSharedMemoryGossipSubscribe?: boolean; + syncMode?: 'on-demand' | 'always-on'; + }): void { if (options?.trackSyncScope !== false) { this.trackSyncContextGraph(contextGraphId); } + const existing = this.subscribedContextGraphs.get(contextGraphId); + // Opening an already durable graph must never silently downgrade it to a + // process-local subscription. An explicit always-on request may promote an + // existing on-demand subscription, while an omitted mode preserves the + // current lifetime (or the legacy always-on default for a new graph). + const requestedSyncMode = options?.syncMode ?? existing?.syncMode ?? 'always-on'; + const syncMode = existing?.subscribed && (existing.syncMode ?? 'always-on') === 'always-on' + ? 'always-on' + : requestedSyncMode; + const persist = syncMode === 'on-demand' ? false : options?.persist; + // SWM gossip subscribe runs `canReadContextGraph` against the local // `_meta` graph. On a fresh `join-approved` notification the curator // has just written the allowlist into ITS _meta, but the requesting @@ -407,12 +423,16 @@ export class SwmSubstrateMethods extends DKGAgentBase { if (!deferSwmGossip) { this.queueSharedMemoryGossipSubscription(contextGraphId); } - const existing = this.subscribedContextGraphs.get(contextGraphId); - if (!existing?.subscribed) { + if (!existing?.subscribed || existing.syncMode !== syncMode) { this.setContextGraphSubscription( contextGraphId, - { ...existing, subscribed: true, synced: existing?.synced ?? false }, - { persist: options?.persist }, + { + ...existing, + subscribed: true, + synced: existing?.synced ?? false, + syncMode, + }, + { persist }, ); } return; @@ -425,11 +445,15 @@ export class SwmSubstrateMethods extends DKGAgentBase { this.gossip.subscribe(publishTopic); this.gossip.subscribe(appTopic); - const existing = this.subscribedContextGraphs.get(contextGraphId); this.setContextGraphSubscription( contextGraphId, - { ...existing, subscribed: true, synced: existing?.synced ?? false }, - { persist: options?.persist }, + { + ...existing, + subscribed: true, + synced: existing?.synced ?? false, + syncMode, + }, + { persist }, ); this.gossip.onMessage(publishTopic, async (_topic, data, from) => { diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index e99152f1d..b2c3428a4 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -605,9 +605,21 @@ export interface ChatSendResult { // ── Context-graph surface ─────────────────────────────────────────── +/** + * Lifetime of an edge node's active Context Graph synchronization intent. + * + * `on-demand` remains active only for the current process. `always-on` is + * restart-durable through the configured subscription store. Persisted rows + * written before this distinction existed are therefore implicitly + * `always-on` for backward compatibility. + */ +export type ContextGraphSyncMode = 'on-demand' | 'always-on'; + /** Tracks the subscription and sync state of a context graph. */ export interface ContextGraphSub { name?: string; + /** Requested synchronization lifetime; omission retains legacy always-on semantics. */ + syncMode?: ContextGraphSyncMode; /** GossipSub topics are active for this context graph. */ subscribed: boolean; /** Definition triples exist in the local triple store. */ diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 85f8ff7c3..945349926 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -337,6 +337,7 @@ import { type PeerDiagnostics, type ChatSendResult, type ContextGraphSub, + type ContextGraphSyncMode, type ContextGraphDiscoveryMetadata, type ContextGraphDiscoveryOptions, type ContextGraphSubscriptionRecord, @@ -449,6 +450,7 @@ export type { PeerDiagnostics, ChatSendResult, ContextGraphSub, + ContextGraphSyncMode, ContextGraphDiscoveryMetadata, ContextGraphDiscoveryOptions, ContextGraphSubscriptionRecord, diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index f9724c8f7..7c8e6cfe6 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -197,6 +197,7 @@ export { type Rfc64CatalogAccessPolicyAuthorityConfigV1, type DKGAgentACKTransportOptions, type ContextGraphSub, + type ContextGraphSyncMode, type ContextGraphDiscoveryMetadata, type ContextGraphDiscoveryOptions, type PublishOpts, diff --git a/packages/agent/test/agent.part-15.test.ts b/packages/agent/test/agent.part-15.test.ts index 500c28ca7..4c7650ba1 100644 --- a/packages/agent/test/agent.part-15.test.ts +++ b/packages/agent/test/agent.part-15.test.ts @@ -142,6 +142,7 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => await agentB.start(); expect(agentB.getSubscribedContextGraphs().get('persisted-cg')).toMatchObject({ subscribed: true, + syncMode: 'always-on', synced: true, sharedMemorySynced: true, metaSynced: true, @@ -162,6 +163,75 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => }); + it('keeps on-demand subscriptions process-local until explicitly promoted', async () => { + const persisted = new Map(); + const subscriptionStore = { + loadAll: async () => [...persisted.values()], + save: async (record: any) => { + persisted.set(record.id, { ...record }); + }, + delete: async (contextGraphId: string) => { + persisted.delete(contextGraphId); + }, + }; + const agentA = await DKGAgent.create({ + name: 'OnDemandSubscriptionLifetimeA', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + }); + + try { + await agentA.start(); + agentA.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + agentA.markContextGraphSubscriptionState('selected-cg', { + synced: true, + sharedMemorySynced: true, + metaSynced: true, + }); + (agentA as any).persistContextGraphSubscription('selected-cg'); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(agentA.getSubscribedContextGraphs().get('selected-cg')).toMatchObject({ + subscribed: true, + syncMode: 'on-demand', + synced: true, + }); + expect(persisted.has('selected-cg')).toBe(false); + } finally { + await agentA.stop().catch(() => {}); + } + + const agentB = await DKGAgent.create({ + name: 'OnDemandSubscriptionLifetimeB', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + }); + try { + await agentB.start(); + expect(agentB.getSubscribedContextGraphs().get('selected-cg')).toBeUndefined(); + + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'always-on' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(persisted.get('selected-cg')).toMatchObject({ + id: 'selected-cg', + subscribed: true, + synced: false, + }); + expect(agentB.getSubscribedContextGraphs().get('selected-cg')?.syncMode).toBe('always-on'); + + // A later UI open is on-demand, but must not silently downgrade an + // operator's explicit always-on choice. + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + expect(agentB.getSubscribedContextGraphs().get('selected-cg')?.syncMode).toBe('always-on'); + } finally { + await agentB.stop().catch(() => {}); + } + }); + + it('rehydrates persisted subscriptions without forcing sync scope', async () => { const subscriptionStore = { loadAll: async () => [{ diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index c06e4a3e4..cd202d771 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises'; import { basename } from 'node:path'; import type { ContextGraphReconcileResult, + ContextGraphSyncMode, RandomSamplingDisabledReason, } from '@origintrail-official/dkg-agent'; import type { JournalReadResult } from '@origintrail-official/dkg-publisher'; @@ -1489,8 +1490,12 @@ export class ApiClient { return this.post('/api/query-remote', { peerId, ...request }); } - async subscribeToContextGraph(contextGraphId: string, options?: { includeSharedMemory?: boolean }): Promise<{ + async subscribeToContextGraph(contextGraphId: string, options?: { + includeSharedMemory?: boolean; + syncMode?: ContextGraphSyncMode; + }): Promise<{ subscribed: string; + syncMode: ContextGraphSyncMode; catchup?: | { connectedPeers: number; @@ -1552,7 +1557,11 @@ export class ApiClient { jobId: string; }; }> { - return this.post('/api/context-graph/subscribe', { contextGraphId, includeWorkspace: options?.includeSharedMemory }); + return this.post('/api/context-graph/subscribe', { + contextGraphId, + includeWorkspace: options?.includeSharedMemory, + syncMode: options?.syncMode, + }); } /** @@ -1566,6 +1575,7 @@ export class ApiClient { /** @deprecated Use subscribeToContextGraph */ async subscribe(contextGraphId: string, options?: { includeWorkspace?: boolean }): Promise<{ subscribed: string; + syncMode: ContextGraphSyncMode; catchup?: | { connectedPeers: number; diff --git a/packages/cli/src/commands/knowledge.ts b/packages/cli/src/commands/knowledge.ts index 5d1a353e7..55c84f13e 100644 --- a/packages/cli/src/commands/knowledge.ts +++ b/packages/cli/src/commands/knowledge.ts @@ -342,8 +342,15 @@ program .action(async (contextGraph: string, opts: ActionOpts) => { try { const client = await ApiClient.connect(); - const result = await client.subscribeToContextGraph(contextGraph); + const result = await client.subscribeToContextGraph(contextGraph, { + syncMode: opts.save ? 'always-on' : 'on-demand', + }); console.log(`Subscribed to context graph: ${contextGraph}`); + console.log( + opts.save + ? 'Synchronization mode: always on (restored after restart).' + : 'Synchronization mode: on demand (current node process only).', + ); const catchup = result.catchup; if (catchup) { if ('peersTried' in catchup) { diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 947e27e15..3681e2294 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -1685,6 +1685,12 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise { }); describe('POST endpoints', () => { + it('subscribeToContextGraph() forwards explicit sync lifetime', async () => { + const { fetch, calls } = createTrackingFetch({ + ok: true, + status: 200, + body: { subscribed: 'cg-selected', syncMode: 'on-demand' }, + }); + globalThis.fetch = fetch; + + await client.subscribeToContextGraph('cg-selected', { + includeSharedMemory: true, + syncMode: 'on-demand', + }); + + expect(calls[0].url).toBe(`http://127.0.0.1:${PORT}/api/context-graph/subscribe`); + expect(JSON.parse(calls[0].opts.body as string)).toEqual({ + contextGraphId: 'cg-selected', + includeWorkspace: true, + syncMode: 'on-demand', + }); + }); + it('sendChat() sends correct body', async () => { const { fetch, calls } = createTrackingFetch({ ok: true, status: 200, body: { delivered: true } }); globalThis.fetch = fetch; diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index ad47313dd..44894c4bb 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -145,6 +145,7 @@ describe('context graph subscribe readiness requires authoritative metadata', () callerAddress?: string; result?: CatchupJobResult; includeSharedMemory?: boolean; + syncMode?: unknown; readiness?: { version: number; durableVerified: boolean; @@ -153,9 +154,14 @@ describe('context graph subscribe readiness requires authoritative metadata', () }; }): Promise<{ response: any; + responseStatus: number; job: any; runCalls: number; runRequests: CatchupRunRequest[]; + subscribeCalls: Array<{ + id: string; + options: { syncMode?: 'on-demand' | 'always-on' } | undefined; + }>; state: Record; patches: Array>; readiness: Record | undefined; @@ -171,6 +177,10 @@ describe('context graph subscribe readiness requires authoritative metadata', () }; let runCalls = 0; const runRequests: CatchupRunRequest[] = []; + const subscribeCalls: Array<{ + id: string; + options: { syncMode?: 'on-demand' | 'always-on' } | undefined; + }> = []; let readiness = opts.readiness ? { ...opts.readiness, updatedAt: opts.readiness.updatedAt ?? Date.now() } : undefined; @@ -187,10 +197,15 @@ describe('context graph subscribe readiness requires authoritative metadata', () const agent = { getContextGraphAllowedAgents: async () => opts.allowedAgents ?? [], getSubscribedContextGraphs: () => state, - subscribeToContextGraph: (id: string) => { + subscribeToContextGraph: ( + id: string, + options?: { syncMode?: 'on-demand' | 'always-on' }, + ) => { + subscribeCalls.push({ id, options }); state.set(id, { ...state.get(id), subscribed: true, + syncMode: options?.syncMode, }); }, markContextGraphSubscriptionState: (id: string, patch: Record) => { @@ -263,26 +278,30 @@ describe('context graph subscribe readiness requires authoritative metadata', () body: JSON.stringify({ contextGraphId, includeSharedMemory: opts.includeSharedMemory ?? true, + ...(opts.syncMode !== undefined ? { syncMode: opts.syncMode } : {}), }), }); const response = await httpResponse.json() as any; - const jobId = response.catchup.jobId as string; + const jobId = response.catchup?.jobId as string | undefined; - for (let i = 0; i < 50; i++) { + for (let i = 0; jobId && i < 50; i++) { if (catchupTracker.jobs.get(jobId)?.finishedAt) break; await new Promise((resolve) => setTimeout(resolve, 5)); } - const statusHttpResponse = await fetch( - `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, - ); - const statusResponse = await statusHttpResponse.json() as any; + const statusResponse = jobId + ? await fetch( + `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, + ).then((result) => result.json()) + : null; return { response, - job: catchupTracker.jobs.get(jobId), + responseStatus: httpResponse.status, + job: jobId ? catchupTracker.jobs.get(jobId) : undefined, runCalls, runRequests, + subscribeCalls, state: state.get(contextGraphId) ?? {}, patches, readiness, @@ -290,6 +309,43 @@ describe('context graph subscribe readiness requires authoritative metadata', () }; } + it('keeps omitted sync mode backward-compatible as always-on', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + }); + + expect(result.response.syncMode).toBe('always-on'); + expect(result.subscribeCalls).toEqual([ + { id: expect.any(String), options: { syncMode: 'always-on' } }, + ]); + expect(result.state.syncMode).toBe('always-on'); + }); + + it('forwards explicit on-demand edge intent without making it always-on', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + syncMode: 'on-demand', + }); + + expect(result.response.syncMode).toBe('on-demand'); + expect(result.subscribeCalls).toEqual([ + { id: expect.any(String), options: { syncMode: 'on-demand' } }, + ]); + expect(result.state.syncMode).toBe('on-demand'); + }); + + it('rejects unknown sync modes before changing subscription state', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + syncMode: 'sometimes', + }); + + expect(result.responseStatus).toBe(400); + expect(result.response.error).toContain('Invalid "syncMode"'); + expect(result.subscribeCalls).toEqual([]); + expect(result.runCalls).toBe(0); + }); + it('does not turn a clean empty response with no authoritative metadata into ready state', async () => { const result = await subscribe({ hasConfirmedMeta: false, diff --git a/packages/node-ui/src/ui/api.ts b/packages/node-ui/src/ui/api.ts index f2057bf6b..520c60479 100644 --- a/packages/node-ui/src/ui/api.ts +++ b/packages/node-ui/src/ui/api.ts @@ -2861,8 +2861,18 @@ export const shutdownNode = () => post<{ ok: boolean }>('/api/shutdown', {}); // --- Integrations --- -export const subscribeToContextGraph = (contextGraphId: string) => - post<{ subscribed: string; catchup?: { status: string; jobId: string } }>('/api/subscribe', { contextGraphId }); +export const subscribeToContextGraph = ( + contextGraphId: string, + options?: { syncMode?: 'on-demand' | 'always-on' }, +) => + post<{ + subscribed: string; + syncMode: 'on-demand' | 'always-on'; + catchup?: { status: string; jobId: string }; + }>('/api/subscribe', { + contextGraphId, + syncMode: options?.syncMode ?? 'on-demand', + }); // --- Notifications (scoped pane wire contract — implementation-plan §3) --- // diff --git a/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx b/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx index 33fd71c30..bf6e8c88d 100644 --- a/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx +++ b/packages/node-ui/src/ui/components/Modals/JoinProjectModal.tsx @@ -325,6 +325,7 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP const [phase, setPhase] = useState('idle'); const [error, setError] = useState(null); const [pendingCgId, setPendingCgId] = useState(null); + const [keepSynced, setKeepSynced] = useState(false); // Phase 8: after approval we transition into a wire-workspace step so // the joiner can populate a local Cursor workspace from the project's // manifest. `wiredCgId` flips the modal into the WireWorkspacePanel; @@ -345,6 +346,7 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP setError(null); setPhase('idle'); setPendingCgId(null); + setKeepSynced(false); } }, [open, initialContextGraphId]); @@ -436,7 +438,9 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP // connection before the daemon snapshots its preferred/fallback peer // cohort for the catch-up job. Dial failure remains best-effort. await warmInviteCuratorConnection(intent.invite); - await subscribeToContextGraph(cgId); + await subscribeToContextGraph(cgId, { + syncMode: keepSynced ? 'always-on' : 'on-demand', + }); await refreshAndOpenContextGraph(cgId, existing); onClose(); } catch (err: unknown) { @@ -636,6 +640,27 @@ export function JoinProjectModal({ open, onClose, initialContextGraphId }: JoinP spellCheck={false} /> + + {intent.kind === 'publicSubscribe' && ( + + )}
diff --git a/packages/node-ui/test/join-project-modal.interaction.test.ts b/packages/node-ui/test/join-project-modal.interaction.test.ts index 3a61ef0e7..620772414 100644 --- a/packages/node-ui/test/join-project-modal.interaction.test.ts +++ b/packages/node-ui/test/join-project-modal.interaction.test.ts @@ -83,6 +83,7 @@ describe('JoinProjectModal public subscription interaction', () => { subscribeToContextGraphMock.mockResolvedValue({ subscribed: 'open-project', + syncMode: 'on-demand', catchup: { status: 'queued', jobId: 'catchup-1' }, }); fetchContextGraphsMock.mockResolvedValue({ @@ -148,7 +149,9 @@ describe('JoinProjectModal public subscription interaction', () => { await act(async () => { primary.click(); }); await flush(); - expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project'); + expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project', { + syncMode: 'on-demand', + }); expect(fetchContextGraphsMock).toHaveBeenCalled(); expect(signJoinRequestMock).not.toHaveBeenCalled(); expect(submitJoinRequestMock).not.toHaveBeenCalled(); @@ -167,11 +170,30 @@ describe('JoinProjectModal public subscription interaction', () => { await flush(); expect(connectToPeerIdWithTimeoutMock).toHaveBeenCalledWith(curatorPeerId); - expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project'); + expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project', { + syncMode: 'on-demand', + }); expect(connectToPeerIdWithTimeoutMock.mock.invocationCallOrder[0]).toBeLessThan( subscribeToContextGraphMock.mock.invocationCallOrder[0], ); expect(signJoinRequestMock).not.toHaveBeenCalled(); expect(submitJoinRequestMock).not.toHaveBeenCalled(); }); + + it('makes restart-durable synchronization an explicit public-graph choice', async () => { + const { container } = await renderModal('open-project'); + const keepSynced = container.querySelector( + 'input[aria-label="Keep this Context Graph synchronized after restart"]', + ) as HTMLInputElement; + expect(keepSynced.checked).toBe(false); + + await act(async () => { keepSynced.click(); }); + const primary = container.querySelector('.v10-modal-btn.primary') as HTMLButtonElement; + await act(async () => { primary.click(); }); + await flush(); + + expect(subscribeToContextGraphMock).toHaveBeenCalledWith('open-project', { + syncMode: 'always-on', + }); + }); }); diff --git a/packages/node-ui/test/ui-api-pure.test.ts b/packages/node-ui/test/ui-api-pure.test.ts index 0a00cf73a..237a62a8d 100644 --- a/packages/node-ui/test/ui-api-pure.test.ts +++ b/packages/node-ui/test/ui-api-pure.test.ts @@ -750,6 +750,14 @@ describe('UI API tests', () => { const call = requestLog.find(r => r.method === 'POST' && r.url.includes('/api/subscribe')); const body = JSON.parse(call?.body ?? '{}'); expect(body.contextGraphId).toBe('cg-1'); + expect(body.syncMode).toBe('on-demand'); + }); + + it('subscribeToContextGraph forwards an explicit always-on choice', async () => { + await subscribeToContextGraph('cg-1', { syncMode: 'always-on' }); + const call = requestLog.find(r => r.method === 'POST' && r.url.includes('/api/subscribe')); + const body = JSON.parse(call?.body ?? '{}'); + expect(body).toEqual({ contextGraphId: 'cg-1', syncMode: 'always-on' }); }); }); From 15098dab5fca77a06f331d15c7e18b7f67b6048a Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 03:23:34 +0200 Subject: [PATCH 02/23] fix(sync): preserve core hosting with on-demand subscriptions --- packages/agent/src/dkg-agent-lifecycle.ts | 64 +++++++++------ packages/agent/src/dkg-agent-swm-host.ts | 3 +- packages/agent/src/dkg-agent-swm-substrate.ts | 12 +-- packages/agent/src/dkg-agent-types.ts | 15 +++- packages/agent/src/dkg-agent.ts | 1 + packages/agent/src/gossip-publish-handler.ts | 16 +++- packages/agent/test/agent.part-15.test.ts | 80 +++++++++++++++++++ .../cli/src/daemon/routes/context-graph.ts | 14 ++-- .../context-graph-subscribe-readiness.test.ts | 33 +++++++- .../test/daemon-http-behavior-extra.test.ts | 17 +++- .../test/knowledge-subscribe-command.test.ts | 77 ++++++++++++++++++ 11 files changed, 284 insertions(+), 48 deletions(-) create mode 100644 packages/cli/test/knowledge-subscribe-command.test.ts diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 3642eef5a..be119b69c 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -457,6 +457,7 @@ import { type PeerDiagnostics, type ChatSendResult, type ContextGraphSub, + type ContextGraphSubInput, type ContextGraphSubscriptionRecord, type ContextGraphSubscriptionRehydrationStatus, type ContextGraphSubscriptionStore, @@ -2674,6 +2675,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { } const approvedSubscription: ContextGraphSub = { ...this.subscribedContextGraphs.get(contextGraphId), + syncMode: 'always-on', subscribed: true, pendingMeta: true, metaSynced: false, @@ -6496,7 +6498,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { setContextGraphSubscription(this: DKGAgent, contextGraphId: string, - next: ContextGraphSub, + next: ContextGraphSubInput, options?: { persist?: boolean; updateRehydrationStatus?: boolean }, ): ContextGraphSub { this.invalidateListContextGraphsCache(); @@ -6512,10 +6514,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { ? this.contextGraphWireId(next.onChainHash) : undefined; const nextWireId = nextOnChainHash ?? localWireId; - const inheritedSyncMode = next.syncMode ?? previous?.syncMode; - const canonicalNext = { + const canonicalNext: ContextGraphSub = { ...next, - ...(inheritedSyncMode ? { syncMode: inheritedSyncMode } : {}), + syncMode: next.syncMode ?? previous?.syncMode ?? 'always-on', ...(next.onChainHash === nextOnChainHash ? {} : { onChainHash: nextOnChainHash }), }; if ( @@ -6529,11 +6530,14 @@ export class LifecycleSyncMethods extends DKGAgentBase { if (!canonicalNext.subscribed && !canonicalNext.coreHosted) { this.clearVmReconcileStateForContextGraph(contextGraphId); } - // On-demand subscriptions deliberately keep their live state and - // readiness process-local. This guard also covers later state patches - // (for example catch-up completion), so they cannot accidentally create a - // durable row after the initial subscribe call opted out. - if (options?.persist !== false && canonicalNext.syncMode !== 'on-demand') { + // On-demand member subscriptions deliberately keep their live state and + // readiness process-local. A Core's independent hosting obligation is + // still durable, though: persistContextGraphSubscription writes a + // host-only snapshot without converting the member intent to always-on. + if ( + options?.persist !== false && + (canonicalNext.syncMode !== 'on-demand' || canonicalNext.coreHosted === true) + ) { if (this.config.contextGraphSubscriptionStore) { const revision = this.nextContextGraphSubscriptionPersistRevision(contextGraphId); this.persistContextGraphSubscription( @@ -6544,9 +6548,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { }, ); } - if (canonicalNext.subscribed) { + if (canonicalNext.syncMode !== 'on-demand' && canonicalNext.subscribed) { this.persistLocalNodeMembership(contextGraphId); - } else { + } else if (canonicalNext.syncMode !== 'on-demand') { this.deleteContextGraphMember(contextGraphId, 'node', this.peerId); } } @@ -6769,7 +6773,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { return; } const sub = this.subscribedContextGraphs.get(contextGraphId); - if (sub?.syncMode === 'on-demand') { + if (sub?.syncMode === 'on-demand' && sub.coreHosted !== true) { // Some lifecycle paths persist reconciliation watermarks directly // instead of going through setContextGraphSubscription. Preserve the // process-local lifetime at this lowest shared write boundary too. @@ -6802,18 +6806,32 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); return; } + const persistMemberIntent = sub.syncMode !== 'on-demand'; + const persistedSnapshot: ContextGraphSub = persistMemberIntent + ? sub + : { + ...sub, + // Preserve only the independent Core-hosting contract. Member + // activation/readiness stays process-local with the on-demand intent. + syncMode: 'always-on', + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: false, + }; const record = { id: contextGraphId, - name: sub.name, - subscribed: sub.subscribed, - synced: sub.synced, - sharedMemorySynced: sub.sharedMemorySynced, - metaSynced: sub.metaSynced, - onChainId: sub.onChainId, - onChainHash: sub.onChainHash, - lastReconciledOrdinal: sub.lastReconciledOrdinal, - coreHosted: sub.coreHosted, - syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), + name: persistedSnapshot.name, + subscribed: persistedSnapshot.subscribed, + synced: persistedSnapshot.synced, + sharedMemorySynced: persistedSnapshot.sharedMemorySynced, + metaSynced: persistedSnapshot.metaSynced, + onChainId: persistedSnapshot.onChainId, + onChainHash: persistedSnapshot.onChainHash, + lastReconciledOrdinal: persistedSnapshot.lastReconciledOrdinal, + coreHosted: persistedSnapshot.coreHosted, + syncScoped: + persistMemberIntent && (this.config.syncContextGraphs ?? []).includes(contextGraphId), }; void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.save(record)) .then(() => { @@ -6821,7 +6839,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { options?.updateRehydrationStatus === true && this.claimContextGraphSubscriptionPersistRevision(contextGraphId, options.revision) ) { - this.updateContextGraphSubscriptionRehydrationStatusAfterPersist(contextGraphId, sub); + this.updateContextGraphSubscriptionRehydrationStatusAfterPersist(contextGraphId, persistedSnapshot); } }).catch((err) => { this.log.warn( diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index bfbcac344..5dc545df2 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -352,6 +352,7 @@ import { type PeerDiagnostics, type ChatSendResult, type ContextGraphSub, + type ContextGraphSubInput, type ContextGraphSubscriptionRecord, type ContextGraphSubscriptionStore, type ContextGraphMemberPrincipalType, @@ -2437,7 +2438,7 @@ export class SwmHostModeMethods extends DKGAgentBase { const existing = this.subscribedContextGraphs.get(localCgId); if (existing?.coreHosted && existing.onChainId === numericStr) return; // already recorded - let next: ContextGraphSub; + let next: ContextGraphSubInput; if (existing) { // Rebind through the helper so a CG re-created/rebound under the same // local id drops its stale reconcile watermark + in-memory cursor before diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index f1549c48b..29fdbc114 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -389,7 +389,7 @@ export class SwmSubstrateMethods extends DKGAgentBase { persist?: boolean; deferSharedMemoryGossipSubscribe?: boolean; syncMode?: 'on-demand' | 'always-on'; - }): void { + }): ContextGraphSub { if (options?.trackSyncScope !== false) { this.trackSyncContextGraph(contextGraphId); } @@ -400,7 +400,7 @@ export class SwmSubstrateMethods extends DKGAgentBase { // existing on-demand subscription, while an omitted mode preserves the // current lifetime (or the legacy always-on default for a new graph). const requestedSyncMode = options?.syncMode ?? existing?.syncMode ?? 'always-on'; - const syncMode = existing?.subscribed && (existing.syncMode ?? 'always-on') === 'always-on' + const syncMode = existing?.subscribed && existing.syncMode === 'always-on' ? 'always-on' : requestedSyncMode; const persist = syncMode === 'on-demand' ? false : options?.persist; @@ -424,7 +424,7 @@ export class SwmSubstrateMethods extends DKGAgentBase { this.queueSharedMemoryGossipSubscription(contextGraphId); } if (!existing?.subscribed || existing.syncMode !== syncMode) { - this.setContextGraphSubscription( + return this.setContextGraphSubscription( contextGraphId, { ...existing, @@ -435,7 +435,7 @@ export class SwmSubstrateMethods extends DKGAgentBase { { persist }, ); } - return; + return existing; } this.gossipRegistered.add(contextGraphId); @@ -445,7 +445,7 @@ export class SwmSubstrateMethods extends DKGAgentBase { this.gossip.subscribe(publishTopic); this.gossip.subscribe(appTopic); - this.setContextGraphSubscription( + const subscription = this.setContextGraphSubscription( contextGraphId, { ...existing, @@ -478,6 +478,8 @@ export class SwmSubstrateMethods extends DKGAgentBase { const fh = this.getOrCreateFinalizationHandler(); await fh.handleFinalizationMessage(data, contextGraphId, from); }); + + return subscription; } /** diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index b2c3428a4..090f97e80 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -618,8 +618,8 @@ export type ContextGraphSyncMode = 'on-demand' | 'always-on'; /** Tracks the subscription and sync state of a context graph. */ export interface ContextGraphSub { name?: string; - /** Requested synchronization lifetime; omission retains legacy always-on semantics. */ - syncMode?: ContextGraphSyncMode; + /** Requested synchronization lifetime, normalized before entering live state. */ + syncMode: ContextGraphSyncMode; /** GossipSub topics are active for this context graph. */ subscribed: boolean; /** Definition triples exist in the local triple store. */ @@ -697,6 +697,17 @@ export interface ContextGraphSub { pendingMeta?: boolean; } +/** + * Compatibility input accepted at subscription-update boundaries. + * + * Historical call sites and persisted records predate synchronization modes, + * so they may omit the field. The agent normalizes this shape exactly once + * before storing it in the live {@link ContextGraphSub} registry. + */ +export type ContextGraphSubInput = Omit & { + syncMode?: ContextGraphSyncMode; +}; + /** * Metadata that passive discovery is allowed to contribute to the local * Context Graph catalogue. diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 945349926..cad08a933 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -1228,6 +1228,7 @@ export class DKGAgent extends DKGAgentBase { const existing = this.subscribedContextGraphs.get(contextGraphId); const next: ContextGraphSub = { ...existing, + syncMode: existing?.syncMode ?? 'always-on', name: metadata.name ?? existing?.name, subscribed: existing?.subscribed === true, synced: existing?.synced === true, diff --git a/packages/agent/src/gossip-publish-handler.ts b/packages/agent/src/gossip-publish-handler.ts index 69db636e4..4038f7e29 100644 --- a/packages/agent/src/gossip-publish-handler.ts +++ b/packages/agent/src/gossip-publish-handler.ts @@ -31,7 +31,11 @@ import { } from '@origintrail-official/dkg-publisher'; import { ethers } from 'ethers'; import type { ContextGraphMetaRecord } from './context-graph-meta-projection.js'; -import type { ContextGraphDiscoveryMetadata, ContextGraphSub } from './dkg-agent-types.js'; +import type { + ContextGraphDiscoveryMetadata, + ContextGraphSub, + ContextGraphSubInput, +} from './dkg-agent-types.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; export type GossipPhaseCallback = (phase: string, status: 'start' | 'end') => void; @@ -132,7 +136,7 @@ function resolveGraphScopedPublishRequest( export interface GossipPublishHandlerCallbacks { contextGraphExists: (id: string) => Promise; getContextGraphOwner: (id: string) => Promise; - setContextGraphSubscription?: (id: string, next: ContextGraphSub, options?: { persist?: boolean }) => void; + setContextGraphSubscription?: (id: string, next: ContextGraphSubInput, options?: { persist?: boolean }) => void; /** * Record a Context Graph learned from ontology gossip. Agent-backed callers * apply the node-role policy centrally (edge=catalogue-only, core=activate); @@ -195,7 +199,7 @@ export class GossipPublishHandler { private setContextGraphSubscription( id: string, - next: ContextGraphSub, + next: ContextGraphSubInput, options?: { persist?: boolean }, ): void { const setter = this.callbacks.setContextGraphSubscription; @@ -203,7 +207,11 @@ export class GossipPublishHandler { setter(id, next, options); return; } - this.subscribedContextGraphs.set(id, next); + const previous = this.subscribedContextGraphs.get(id); + this.subscribedContextGraphs.set(id, { + ...next, + syncMode: next.syncMode ?? previous?.syncMode ?? 'always-on', + }); } private recordDiscoveredContextGraph(id: string, metadata: ContextGraphDiscoveryMetadata): void { diff --git a/packages/agent/test/agent.part-15.test.ts b/packages/agent/test/agent.part-15.test.ts index 4c7650ba1..586a30404 100644 --- a/packages/agent/test/agent.part-15.test.ts +++ b/packages/agent/test/agent.part-15.test.ts @@ -232,6 +232,86 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => }); + it('persists a Core hosting obligation without persisting its on-demand member intent', async () => { + const persisted = new Map(); + const subscriptionStore = { + loadAll: async () => [...persisted.values()], + save: async (record: any) => { + persisted.set(record.id, { ...record }); + }, + delete: async (contextGraphId: string) => { + persisted.delete(contextGraphId); + }, + }; + const localCgId = 'selected-hosted-cg'; + + const agentA = await DKGAgent.create({ + name: 'OnDemandCoreHostA', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + nodeRole: 'core', + }); + try { + await agentA.start(); + agentA.subscribeToContextGraph(localCgId, { syncMode: 'on-demand' }); + agentA.markContextGraphSubscriptionState(localCgId, { + synced: true, + sharedMemorySynced: true, + metaSynced: true, + }); + (agentA as any).chain.getContextGraphAccessPolicy = async () => 0; + (agentA as any).chain.isContextGraphActiveOnChain = async () => true; + + await (agentA as any).recordCoreHostedPublicCg('14', localCgId); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(agentA.getSubscribedContextGraphs().get(localCgId)).toMatchObject({ + subscribed: true, + syncMode: 'on-demand', + synced: true, + coreHosted: true, + onChainId: '14', + }); + expect(persisted.get(localCgId)).toMatchObject({ + id: localCgId, + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: false, + coreHosted: true, + onChainId: '14', + syncScoped: false, + }); + } finally { + await agentA.stop().catch(() => {}); + } + + const agentB = await DKGAgent.create({ + name: 'OnDemandCoreHostB', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + nodeRole: 'core', + }); + try { + await agentB.start(); + expect(agentB.getSubscribedContextGraphs().get(localCgId)).toMatchObject({ + subscribed: false, + syncMode: 'always-on', + synced: false, + sharedMemorySynced: false, + metaSynced: false, + coreHosted: true, + onChainId: '14', + }); + expect((agentB as any).config.syncContextGraphs ?? []).not.toContain(localCgId); + } finally { + await agentB.stop().catch(() => {}); + } + }); + + it('rehydrates persisted subscriptions without forcing sync scope', async () => { const subscriptionStore = { loadAll: async () => [{ diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 3681e2294..89031e04d 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -1730,13 +1730,13 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise { subscribeCalls.push({ id, options }); - state.set(id, { - ...state.get(id), + const previous = state.get(id); + const effectiveSyncMode = previous?.subscribed && previous.syncMode === 'always-on' + ? 'always-on' + : options?.syncMode ?? previous?.syncMode ?? 'always-on'; + const applied = { + ...previous, subscribed: true, - syncMode: options?.syncMode, - }); + synced: previous?.synced ?? false, + syncMode: effectiveSyncMode, + }; + state.set(id, applied); + return applied; }, markContextGraphSubscriptionState: (id: string, patch: Record) => { patches.push({ ...patch }); @@ -334,6 +341,24 @@ describe('context graph subscribe readiness requires authoritative metadata', () expect(result.state.syncMode).toBe('on-demand'); }); + it('reports the agent-applied mode when an on-demand open cannot downgrade always-on', async () => { + const result = await subscribe({ + hasConfirmedMeta: false, + syncMode: 'on-demand', + initial: { + subscribed: true, + syncMode: 'always-on', + synced: false, + }, + }); + + expect(result.subscribeCalls).toEqual([ + { id: expect.any(String), options: { syncMode: 'on-demand' } }, + ]); + expect(result.response.syncMode).toBe('always-on'); + expect(result.state.syncMode).toBe('always-on'); + }); + it('rejects unknown sync modes before changing subscription state', async () => { const result = await subscribe({ hasConfirmedMeta: false, diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index 951b3925e..baee38636 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -1051,7 +1051,11 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { const agent = { getContextGraphAllowedAgents: async () => [], getSubscribedContextGraphs: () => new Map(), - subscribeToContextGraph: () => {}, + subscribeToContextGraph: () => ({ + subscribed: true, + synced: false, + syncMode: 'always-on' as const, + }), contextGraphHasLocalContent: async () => false, markContextGraphSubscriptionState: () => { throw new Error('timeout-only catchup must not mark subscription synced'); @@ -1166,7 +1170,11 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { const agent = { getContextGraphAllowedAgents: async () => [], getSubscribedContextGraphs: () => new Map(), - subscribeToContextGraph: () => {}, + subscribeToContextGraph: () => ({ + subscribed: true, + synced: false, + syncMode: 'always-on' as const, + }), contextGraphHasLocalContent: async () => true, markContextGraphSubscriptionState: () => { markedSynced = true; }, resolveAgentByToken: () => undefined, @@ -2300,6 +2308,11 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli getSubscribedContextGraphs: () => new Map(), subscribeToContextGraph: () => { subscribeCalled = true; + return { + subscribed: true, + synced: false, + syncMode: 'always-on' as const, + }; }, contextGraphHasLocalContent: async () => false, markContextGraphSubscriptionState: () => {}, diff --git a/packages/cli/test/knowledge-subscribe-command.test.ts b/packages/cli/test/knowledge-subscribe-command.test.ts new file mode 100644 index 000000000..4588aefcf --- /dev/null +++ b/packages/cli/test/knowledge-subscribe-command.test.ts @@ -0,0 +1,77 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { Command } from 'commander'; + +const configMocks = vi.hoisted(() => ({ + loadConfig: vi.fn(async () => ({ contextGraphs: [] as string[] })), + saveConfig: vi.fn(async () => undefined), + resolveContextGraphs: vi.fn((config: { contextGraphs?: string[] }) => config.contextGraphs ?? []), +})); + +vi.mock('../src/config.js', async (importOriginal) => ({ + ...await importOriginal(), + loadConfig: configMocks.loadConfig, + saveConfig: configMocks.saveConfig, + resolveContextGraphs: configMocks.resolveContextGraphs, +})); + +import { ApiClient } from '../src/api-client.js'; +import { registerKnowledgeCommands } from '../src/commands/knowledge.js'; + +function commandProgram(): Command { + const program = new Command().name('dkg'); + program.exitOverride(); + registerKnowledgeCommands(program); + return program; +} + +describe('knowledge subscribe CLI sync lifetime', () => { + const logLines: string[] = []; + + beforeEach(() => { + logLines.length = 0; + configMocks.loadConfig.mockClear(); + configMocks.saveConfig.mockClear(); + configMocks.resolveContextGraphs.mockClear(); + vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + logLines.push(args.map(String).join(' ')); + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('requests process-local on-demand synchronization by default', async () => { + const subscribeToContextGraph = vi.fn().mockResolvedValue({ + subscribed: 'selected-cg', + syncMode: 'on-demand', + }); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ subscribeToContextGraph } as unknown as ApiClient); + + await commandProgram().parseAsync(['node', 'dkg', 'subscribe', 'selected-cg']); + + expect(subscribeToContextGraph).toHaveBeenCalledWith('selected-cg', { + syncMode: 'on-demand', + }); + expect(configMocks.saveConfig).not.toHaveBeenCalled(); + expect(logLines.join('\n')).toContain('Synchronization mode: on demand'); + }); + + it('requests restart-durable synchronization with --save', async () => { + const subscribeToContextGraph = vi.fn().mockResolvedValue({ + subscribed: 'selected-cg', + syncMode: 'always-on', + }); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ subscribeToContextGraph } as unknown as ApiClient); + + await commandProgram().parseAsync(['node', 'dkg', 'subscribe', 'selected-cg', '--save']); + + expect(subscribeToContextGraph).toHaveBeenCalledWith('selected-cg', { + syncMode: 'always-on', + }); + expect(configMocks.saveConfig).toHaveBeenCalledWith(expect.objectContaining({ + contextGraphs: ['selected-cg'], + })); + expect(logLines.join('\n')).toContain('Synchronization mode: always on'); + }); +}); From 7f0c0ffb94195dbf6782a942b02ee1a70672dc9c Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 04:03:59 +0200 Subject: [PATCH 03/23] fix(sync): preserve durable intent after on-demand opens --- packages/agent/src/dkg-agent-cg-registry.ts | 4 +- packages/agent/src/dkg-agent-context-graph.ts | 7 +- packages/agent/src/dkg-agent-lifecycle.ts | 48 +++++----- packages/agent/src/dkg-agent-publish.ts | 2 +- packages/agent/src/dkg-agent.ts | 1 + packages/agent/test/agent.part-15.test.ts | 89 +++++++++++++++++++ packages/cli/src/api-client.ts | 5 +- packages/cli/src/commands/knowledge.ts | 2 +- packages/cli/src/daemon/lifecycle.ts | 4 +- packages/cli/test/api-client.test.ts | 17 ++++ .../daemon-context-graph-bootstrap.test.ts | 3 + .../test/knowledge-subscribe-command.test.ts | 17 ++++ 12 files changed, 164 insertions(+), 35 deletions(-) diff --git a/packages/agent/src/dkg-agent-cg-registry.ts b/packages/agent/src/dkg-agent-cg-registry.ts index 115602f5b..d69d8c686 100644 --- a/packages/agent/src/dkg-agent-cg-registry.ts +++ b/packages/agent/src/dkg-agent-cg-registry.ts @@ -1048,7 +1048,7 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { // one curator triple per node and `getContextGraphOwner`'s // `LIMIT 1` made ownership nondeterministic — any subscriber could // win the unordered query and look like the curator. - this.subscribeToContextGraph(opts.id); + this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); this.setContextGraphSubscription(opts.id, { name: opts.name, subscribed: true, @@ -1108,7 +1108,7 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { this.contextGraphMetaProjection.markDirtyFromQuads(quads); await gm.ensureNewContextGraph(opts.id); - this.subscribeToContextGraph(opts.id); + this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); this.setContextGraphSubscription(opts.id, { name: opts.name, subscribed: true, diff --git a/packages/agent/src/dkg-agent-context-graph.ts b/packages/agent/src/dkg-agent-context-graph.ts index 4317e4ba1..322e65ef9 100644 --- a/packages/agent/src/dkg-agent-context-graph.ts +++ b/packages/agent/src/dkg-agent-context-graph.ts @@ -817,7 +817,7 @@ export class ContextGraphMethods extends DKGAgentBase { } if (!opts.private) { - this.subscribeToContextGraph(opts.id); + this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); // Curated CGs: definition lives in _meta, NOT in ONTOLOGY. Do not // broadcast to the network — only invited nodes will discover it via @@ -1483,8 +1483,11 @@ export class ContextGraphMethods extends DKGAgentBase { const next = { ...sub, onChainHash: nameHash }; this.bindSubscriptionOnChainId(id, next, onChainId); this.setContextGraphSubscription(id, next, { persist: false }); + this.subscribeToContextGraph(id, { + trackSyncScope: true, + syncMode: 'always-on', + }); if (!next.subscribed) { - this.subscribeToContextGraph(id, { trackSyncScope: true }); this.log.info(ctx, `Subscribed to newly registered context graph "${id}"`); } this.persistContextGraphSubscription(id); diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index be119b69c..d4d8e8b1c 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -2744,6 +2744,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // update/finalization) wire up immediately as before. this.subscribeToContextGraph(contextGraphId, { deferSharedMemoryGossipSubscribe: true, + syncMode: 'always-on', // The exact approval snapshot was committed above. Scheduling // the ordinary background persistence here would reintroduce // untracked writes around the compensating transaction. @@ -3033,7 +3034,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // Subscribe to both system context graph GossipSub topics for (const systemContextGraph of [SYSTEM_CONTEXT_GRAPHS.AGENTS, SYSTEM_CONTEXT_GRAPHS.ONTOLOGY]) { - this.subscribeToContextGraph(systemContextGraph); + this.subscribeToContextGraph(systemContextGraph, { syncMode: 'always-on' }); } // Connect to bootstrap peers @@ -6638,7 +6639,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { updateContextGraphSubscriptionRehydrationStatusAfterPersist(this: DKGAgent, contextGraphId: string, - next?: ContextGraphSub, + next?: Pick, ): void { const status = this.contextGraphSubscriptionRehydrationStatus; if (!status) return; @@ -6807,29 +6808,20 @@ export class LifecycleSyncMethods extends DKGAgentBase { return; } const persistMemberIntent = sub.syncMode !== 'on-demand'; - const persistedSnapshot: ContextGraphSub = persistMemberIntent - ? sub - : { - ...sub, - // Preserve only the independent Core-hosting contract. Member - // activation/readiness stays process-local with the on-demand intent. - syncMode: 'always-on', - subscribed: false, - synced: false, - sharedMemorySynced: false, - metaSynced: false, - }; - const record = { + // Project the durable record directly from the real live state. An + // on-demand member can coexist with an independent Core-hosting duty, but + // that host-only row must not masquerade as a second live subscription. + const record: ContextGraphSubscriptionRecord = { id: contextGraphId, - name: persistedSnapshot.name, - subscribed: persistedSnapshot.subscribed, - synced: persistedSnapshot.synced, - sharedMemorySynced: persistedSnapshot.sharedMemorySynced, - metaSynced: persistedSnapshot.metaSynced, - onChainId: persistedSnapshot.onChainId, - onChainHash: persistedSnapshot.onChainHash, - lastReconciledOrdinal: persistedSnapshot.lastReconciledOrdinal, - coreHosted: persistedSnapshot.coreHosted, + name: sub.name, + subscribed: persistMemberIntent && sub.subscribed, + synced: persistMemberIntent && sub.synced, + sharedMemorySynced: persistMemberIntent ? sub.sharedMemorySynced : false, + metaSynced: persistMemberIntent ? sub.metaSynced : false, + onChainId: sub.onChainId, + onChainHash: sub.onChainHash, + lastReconciledOrdinal: sub.lastReconciledOrdinal, + coreHosted: sub.coreHosted, syncScoped: persistMemberIntent && (this.config.syncContextGraphs ?? []).includes(contextGraphId), }; @@ -6839,7 +6831,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { options?.updateRehydrationStatus === true && this.claimContextGraphSubscriptionPersistRevision(contextGraphId, options.revision) ) { - this.updateContextGraphSubscriptionRehydrationStatusAfterPersist(contextGraphId, persistedSnapshot); + this.updateContextGraphSubscriptionRehydrationStatusAfterPersist(contextGraphId, record); } }).catch((err) => { this.log.warn( @@ -7309,7 +7301,11 @@ export class LifecycleSyncMethods extends DKGAgentBase { this.trackSyncContextGraph(row.id); } if (row.subscribed) { - this.subscribeToContextGraph(row.id, { trackSyncScope: false, persist: false }); + this.subscribeToContextGraph(row.id, { + trackSyncScope: false, + persist: false, + syncMode: 'always-on', + }); this.persistLocalNodeMembership(row.id, 'rehydrated-subscription'); } // Upgrade/self-heal path for late private-CG members whose payload and diff --git a/packages/agent/src/dkg-agent-publish.ts b/packages/agent/src/dkg-agent-publish.ts index f0a5feac5..cb4882240 100644 --- a/packages/agent/src/dkg-agent-publish.ts +++ b/packages/agent/src/dkg-agent-publish.ts @@ -2495,7 +2495,7 @@ export class PublishMethods extends DKGAgentBase { this.contextGraphMetaProjection.markDirtyFromQuads(quads); await gm.ensureContextGraph(contextGraphId); await this.store.flush?.(); - this.subscribeToContextGraph(contextGraphId); + this.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); this.setContextGraphSubscription(contextGraphId, { ...existingSub, name, diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index cad08a933..dceb4c89e 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -1255,6 +1255,7 @@ export class DKGAgent extends DKGAgentBase { if (!existing && (this.config.nodeRole ?? 'edge') === 'core') { this.subscribeToContextGraph(contextGraphId, { trackSyncScope: options.trackSyncScope, + syncMode: 'always-on', }); } diff --git a/packages/agent/test/agent.part-15.test.ts b/packages/agent/test/agent.part-15.test.ts index 586a30404..d08ed5477 100644 --- a/packages/agent/test/agent.part-15.test.ts +++ b/packages/agent/test/agent.part-15.test.ts @@ -165,6 +165,7 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => it('keeps on-demand subscriptions process-local until explicitly promoted', async () => { const persisted = new Map(); + const persistedMembers = new Map(); const subscriptionStore = { loadAll: async () => [...persisted.values()], save: async (record: any) => { @@ -174,11 +175,20 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => persisted.delete(contextGraphId); }, }; + const membershipStore = { + upsert: async (record: any) => { + persistedMembers.set(`${record.contextGraphId}|${record.principalType}|${record.principalId}`, { ...record }); + }, + delete: async (contextGraphId: string, principalType: string, principalId: string) => { + persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); + }, + }; const agentA = await DKGAgent.create({ name: 'OnDemandSubscriptionLifetimeA', listenHost: '127.0.0.1', chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), contextGraphSubscriptionStore: subscriptionStore, + contextGraphMembershipStore: membershipStore, }); try { @@ -198,6 +208,7 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => synced: true, }); expect(persisted.has('selected-cg')).toBe(false); + expect(persistedMembers.has(`selected-cg|node|${agentA.peerId}`)).toBe(false); } finally { await agentA.stop().catch(() => {}); } @@ -207,12 +218,16 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => listenHost: '127.0.0.1', chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), contextGraphSubscriptionStore: subscriptionStore, + contextGraphMembershipStore: membershipStore, }); try { await agentB.start(); expect(agentB.getSubscribedContextGraphs().get('selected-cg')).toBeUndefined(); agentB.subscribeToContextGraph('selected-cg', { syncMode: 'on-demand' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(persistedMembers.has(`selected-cg|node|${agentB.peerId}`)).toBe(false); + agentB.subscribeToContextGraph('selected-cg', { syncMode: 'always-on' }); await new Promise((resolve) => setTimeout(resolve, 0)); expect(persisted.get('selected-cg')).toMatchObject({ @@ -221,6 +236,13 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => synced: false, }); expect(agentB.getSubscribedContextGraphs().get('selected-cg')?.syncMode).toBe('always-on'); + expect(persistedMembers.get(`selected-cg|node|${agentB.peerId}`)).toMatchObject({ + contextGraphId: 'selected-cg', + principalType: 'node', + principalId: agentB.peerId, + status: 'active', + source: 'subscription', + }); // A later UI open is on-demand, but must not silently downgrade an // operator's explicit always-on choice. @@ -232,6 +254,73 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => }); + it('promotes an existing on-demand selection when the node creates the graph', async () => { + const persisted = new Map(); + const persistedMembers = new Map(); + const subscriptionStore = { + loadAll: async () => [...persisted.values()], + save: async (record: any) => { + persisted.set(record.id, { ...record }); + }, + delete: async (contextGraphId: string) => { + persisted.delete(contextGraphId); + }, + }; + const membershipStore = { + upsert: async (record: any) => { + persistedMembers.set(`${record.contextGraphId}|${record.principalType}|${record.principalId}`, { ...record }); + }, + delete: async (contextGraphId: string, principalType: string, principalId: string) => { + persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); + }, + }; + const contextGraphId = 'selected-then-created-cg'; + const agent = await DKGAgent.create({ + name: 'OnDemandCreatePromotion', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: subscriptionStore, + contextGraphMembershipStore: membershipStore, + }); + + try { + await agent.start(); + agent.subscribeToContextGraph(contextGraphId, { syncMode: 'on-demand' }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(persisted.has(contextGraphId)).toBe(false); + + await agent.createContextGraph({ + id: contextGraphId, + name: 'Selected then created', + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(agent.getSubscribedContextGraphs().get(contextGraphId)).toMatchObject({ + subscribed: true, + syncMode: 'always-on', + synced: true, + metaSynced: true, + }); + expect(persisted.get(contextGraphId)).toMatchObject({ + id: contextGraphId, + subscribed: true, + synced: true, + metaSynced: true, + syncScoped: true, + }); + expect(persistedMembers.get(`${contextGraphId}|node|${agent.peerId}`)).toMatchObject({ + contextGraphId, + principalType: 'node', + principalId: agent.peerId, + status: 'active', + source: 'subscription', + }); + } finally { + await agent.stop().catch(() => {}); + } + }); + + it('persists a Core hosting obligation without persisting its on-demand member intent', async () => { const persisted = new Map(); const subscriptionStore = { diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index cd202d771..054d89bef 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -1637,7 +1637,10 @@ export class ApiClient { jobId: string; }; }> { - return this.subscribeToContextGraph(contextGraphId, { includeSharedMemory: options?.includeWorkspace }); + return this.subscribeToContextGraph(contextGraphId, { + includeSharedMemory: options?.includeWorkspace, + syncMode: 'always-on', + }); } async catchupStatus(contextGraphId: string): Promise<{ diff --git a/packages/cli/src/commands/knowledge.ts b/packages/cli/src/commands/knowledge.ts index 55c84f13e..c3177ad91 100644 --- a/packages/cli/src/commands/knowledge.ts +++ b/packages/cli/src/commands/knowledge.ts @@ -347,7 +347,7 @@ program }); console.log(`Subscribed to context graph: ${contextGraph}`); console.log( - opts.save + result.syncMode === 'always-on' ? 'Synchronization mode: always on (restored after restart).' : 'Synchronization mode: on demand (current node process only).', ); diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index b0fae0569..4760c8efe 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -1072,13 +1072,13 @@ export async function bootstrapConfiguredContextGraphs(input: { input.log( `Context graph "${contextGraphId}" setup failed: ${err instanceof Error ? err.message : String(err)} — will discover via sync/gossip`, ); - input.agent.subscribeToContextGraph(contextGraphId); + input.agent.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); } continue; } const existing = input.agent.getSubscribedContextGraphs().get(contextGraphId); - input.agent.subscribeToContextGraph(contextGraphId); + input.agent.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); let hasAuthoritativeMetadata = false; let locallyCurated = false; diff --git a/packages/cli/test/api-client.test.ts b/packages/cli/test/api-client.test.ts index 7768b86eb..22b5de629 100644 --- a/packages/cli/test/api-client.test.ts +++ b/packages/cli/test/api-client.test.ts @@ -480,6 +480,23 @@ describe('ApiClient', () => { }); }); + it('subscribe() keeps the legacy restart-durable lifetime explicit', async () => { + const { fetch, calls } = createTrackingFetch({ + ok: true, + status: 200, + body: { subscribed: 'cg-legacy', syncMode: 'always-on' }, + }); + globalThis.fetch = fetch; + + await client.subscribe('cg-legacy', { includeWorkspace: true }); + + expect(JSON.parse(calls[0].opts.body as string)).toEqual({ + contextGraphId: 'cg-legacy', + includeWorkspace: true, + syncMode: 'always-on', + }); + }); + it('sendChat() sends correct body', async () => { const { fetch, calls } = createTrackingFetch({ ok: true, status: 200, body: { delivered: true } }); globalThis.fetch = fetch; diff --git a/packages/cli/test/daemon-context-graph-bootstrap.test.ts b/packages/cli/test/daemon-context-graph-bootstrap.test.ts index fe9583e47..04cff4a1e 100644 --- a/packages/cli/test/daemon-context-graph-bootstrap.test.ts +++ b/packages/cli/test/daemon-context-graph-bootstrap.test.ts @@ -84,6 +84,7 @@ describe('configured context graph daemon bootstrap', () => { expect(fixture.ensureContextGraphLocal).not.toHaveBeenCalled(); expect(fixture.subscribeToContextGraph).toHaveBeenCalledWith( '0x1234567890123456789012345678901234567890/private-cg', + { syncMode: 'always-on' }, ); expect( fixture.subscriptions.get('0x1234567890123456789012345678901234567890/private-cg'), @@ -148,6 +149,7 @@ describe('configured context graph daemon bootstrap', () => { expect(fixture.subscribeToContextGraph).toHaveBeenCalledWith( '0x1234567890123456789012345678901234567890/local-cg', + { syncMode: 'always-on' }, ); expect(fixture.markContextGraphSubscriptionState).not.toHaveBeenCalled(); expect( @@ -434,6 +436,7 @@ describe('configured context graph daemon bootstrap', () => { expect(fixture.subscribeToContextGraph).toHaveBeenCalledTimes(1); expect(fixture.subscribeToContextGraph).toHaveBeenCalledWith( '0x1234567890123456789012345678901234567890/remote-cg', + { syncMode: 'always-on' }, ); expect(fixture.subscriptions.has(SYSTEM_CONTEXT_GRAPHS.AGENTS)).toBe(false); expect(fixture.subscriptions.has(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY)).toBe(false); diff --git a/packages/cli/test/knowledge-subscribe-command.test.ts b/packages/cli/test/knowledge-subscribe-command.test.ts index 4588aefcf..52c865ad8 100644 --- a/packages/cli/test/knowledge-subscribe-command.test.ts +++ b/packages/cli/test/knowledge-subscribe-command.test.ts @@ -74,4 +74,21 @@ describe('knowledge subscribe CLI sync lifetime', () => { })); expect(logLines.join('\n')).toContain('Synchronization mode: always on'); }); + + it('reports the server-normalized mode when an on-demand request stays always-on', async () => { + const subscribeToContextGraph = vi.fn().mockResolvedValue({ + subscribed: 'selected-cg', + syncMode: 'always-on', + }); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ subscribeToContextGraph } as unknown as ApiClient); + + await commandProgram().parseAsync(['node', 'dkg', 'subscribe', 'selected-cg']); + + expect(subscribeToContextGraph).toHaveBeenCalledWith('selected-cg', { + syncMode: 'on-demand', + }); + expect(configMocks.saveConfig).not.toHaveBeenCalled(); + expect(logLines.join('\n')).toContain('Synchronization mode: always on'); + expect(logLines.join('\n')).not.toContain('Synchronization mode: on demand'); + }); }); From 910161f420ae1cde6c518a23638ec74bfe58a24b Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 04:36:46 +0200 Subject: [PATCH 04/23] test(sync): preserve registration subscription invariant --- packages/agent/test/core-fills-gap.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/agent/test/core-fills-gap.test.ts b/packages/agent/test/core-fills-gap.test.ts index 495da486d..ba7996bfa 100644 --- a/packages/agent/test/core-fills-gap.test.ts +++ b/packages/agent/test/core-fills-gap.test.ts @@ -93,7 +93,8 @@ interface AgentInternals { watermarkAfter: number; }>; runVmReconcileSweep(): Promise; - subscribedContextGraphs: Map; + subscribedContextGraphs: Map; + gossipRegistered: Set; vmReconcileDispatcher: { triggerLive: (cg: string) => void; triggerPeriodic: (cg: string) => void; @@ -631,7 +632,7 @@ describe('Phase D — recordCoreHostedPublicCg', () => { expect(((internals as any).recentReconciledUals as { has(key: string): boolean }).has(recentKey)).toBe(false); }); - it('clears VM reconcile state when stale inactive on-chain ids are re-registered', async () => { + it('clears VM reconcile state and promotes durable mode when stale inactive on-chain ids are re-registered', async () => { const internals = await boot(); const localCgId = 'stale-register'; const ownerAddr = (internals.chain as unknown as { signerAddress: string }).signerAddress; @@ -648,9 +649,13 @@ describe('Phase D — recordCoreHostedPublicCg', () => { object: '"5"', graph: contextGraphDataGraphUri(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY), }]); + // This low-level fixture intentionally has no gossip runtime. Model a + // valid already-live on-demand member, including the handler-registration + // invariant that makes subscribeToContextGraph's promotion path idempotent. internals.subscribedContextGraphs.set(localCgId, { - subscribed: true, onChainId: '5', lastReconciledOrdinal: 4, + syncMode: 'on-demand', subscribed: true, onChainId: '5', lastReconciledOrdinal: 4, }); + internals.gossipRegistered.add(localCgId); internals.chain.isContextGraphActiveOnChain = async (id) => id !== 5n; const storageAddr = await internals.chain.getDKGKnowledgeAssetsAddress(); @@ -685,6 +690,7 @@ describe('Phase D — recordCoreHostedPublicCg', () => { const sub = internals.subscribedContextGraphs.get(localCgId); expect(sub?.onChainId).not.toBe('5'); + expect(sub?.syncMode).toBe('always-on'); expect(sub?.lastReconciledOrdinal).toBe(0); expect(recent.has(recentKey)).toBe(false); expect(negativeCache.has(recentKey)).toBe(false); From d84b1f42c325642e988843beefb9e740977df04a Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 04:38:56 +0200 Subject: [PATCH 05/23] fix(sync): retain implicit graph durable promotion --- packages/agent/src/dkg-agent-publish.ts | 4 ++-- packages/agent/test/discovery-subscription-boundary.test.ts | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/dkg-agent-publish.ts b/packages/agent/src/dkg-agent-publish.ts index cb4882240..7967c1a23 100644 --- a/packages/agent/src/dkg-agent-publish.ts +++ b/packages/agent/src/dkg-agent-publish.ts @@ -2495,9 +2495,9 @@ export class PublishMethods extends DKGAgentBase { this.contextGraphMetaProjection.markDirtyFromQuads(quads); await gm.ensureContextGraph(contextGraphId); await this.store.flush?.(); - this.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); + const promotedSub = this.subscribeToContextGraph(contextGraphId, { syncMode: 'always-on' }); this.setContextGraphSubscription(contextGraphId, { - ...existingSub, + ...promotedSub, name, subscribed: true, synced: true, diff --git a/packages/agent/test/discovery-subscription-boundary.test.ts b/packages/agent/test/discovery-subscription-boundary.test.ts index 3ec65ed1e..daf3ed071 100644 --- a/packages/agent/test/discovery-subscription-boundary.test.ts +++ b/packages/agent/test/discovery-subscription-boundary.test.ts @@ -652,11 +652,15 @@ describe('Context Graph discovery/subscription boundary', () => { id: 'explicit-local-create', name: 'Explicit Local Create', }); + agent.subscribeToContextGraph('implicit-local-write', { syncMode: 'on-demand' }); + expect(agent.getSubscribedContextGraphs().get('implicit-local-write')?.syncMode).toBe('on-demand'); + expect(persisted.has('implicit-local-write')).toBe(false); await agent.ensureImplicitSharedMemoryContextGraph('implicit-local-write'); await new Promise((resolve) => setTimeout(resolve, 0)); for (const id of ['explicit-local-create', 'implicit-local-write']) { expect(agent.getSubscribedContextGraphs().get(id)?.subscribed).toBe(true); + expect(agent.getSubscribedContextGraphs().get(id)?.syncMode).toBe('always-on'); expect((agent as any).config.syncContextGraphs ?? []).toContain(id); expect((agent as any).gossipRegistered.has(id)).toBe(true); expect(persisted.get(id)).toMatchObject({ subscribed: true, syncScoped: true }); From 20271bf456ef7549fa460c20bf2b6618eabca8fa Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 04:50:16 +0200 Subject: [PATCH 06/23] test(sync): extract subscription persistence fixtures --- packages/agent/test/agent.part-15.test.ts | 144 ++++++++-------------- 1 file changed, 51 insertions(+), 93 deletions(-) diff --git a/packages/agent/test/agent.part-15.test.ts b/packages/agent/test/agent.part-15.test.ts index d08ed5477..9885a7801 100644 --- a/packages/agent/test/agent.part-15.test.ts +++ b/packages/agent/test/agent.part-15.test.ts @@ -3,6 +3,46 @@ import { describe, it, expect, beforeAll, afterAll, vi, DKGAgentWallet, buildAge let _fileSnapshot: string; + +function createContextGraphPersistenceFixture() { + const persisted = new Map(); + const persistedMembers = new Map(); + const subscriptionStore = { + loadAll: async () => [...persisted.values()], + save: async (record: any) => { + persisted.set(record.id, { ...record }); + }, + delete: async (contextGraphId: string) => { + persisted.delete(contextGraphId); + }, + }; + const membershipStore = { + upsert: async (record: any) => { + persistedMembers.set( + `${record.contextGraphId}|${record.principalType}|${record.principalId}`, + { ...record }, + ); + }, + delete: async (contextGraphId: string, principalType: string, principalId: string) => { + persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); + }, + }; + return { persisted, persistedMembers, subscriptionStore, membershipStore }; +} + +async function createAgentWithContextGraphPersistence( + name: string, + fixture: ReturnType, +) { + return DKGAgent.create({ + name, + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: fixture.subscriptionStore, + contextGraphMembershipStore: fixture.membershipStore, + }); +} + beforeAll(async () => { _fileSnapshot = await takeSnapshot(); const { hubAddress } = getSharedContext(); @@ -68,33 +108,9 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => it('persists runtime subscriptions and rehydrates them on restart', async () => { - const persisted = new Map(); - const persistedMembers = new Map(); - const subscriptionStore = { - loadAll: async () => [...persisted.values()], - save: async (record: any) => { - persisted.set(record.id, { ...record }); - }, - delete: async (contextGraphId: string) => { - persisted.delete(contextGraphId); - }, - }; - const membershipStore = { - upsert: async (record: any) => { - persistedMembers.set(`${record.contextGraphId}|${record.principalType}|${record.principalId}`, { ...record }); - }, - delete: async (contextGraphId: string, principalType: string, principalId: string) => { - persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); - }, - }; - - const agentA = await DKGAgent.create({ - name: 'PersistedSubscriptionsA', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const agentA = await createAgentWithContextGraphPersistence('PersistedSubscriptionsA', fixture); let agentAPeerId = ''; try { @@ -130,13 +146,7 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => source: 'subscription', }); - const agentB = await DKGAgent.create({ - name: 'PersistedSubscriptionsB', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const agentB = await createAgentWithContextGraphPersistence('PersistedSubscriptionsB', fixture); try { await agentB.start(); @@ -164,32 +174,9 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => it('keeps on-demand subscriptions process-local until explicitly promoted', async () => { - const persisted = new Map(); - const persistedMembers = new Map(); - const subscriptionStore = { - loadAll: async () => [...persisted.values()], - save: async (record: any) => { - persisted.set(record.id, { ...record }); - }, - delete: async (contextGraphId: string) => { - persisted.delete(contextGraphId); - }, - }; - const membershipStore = { - upsert: async (record: any) => { - persistedMembers.set(`${record.contextGraphId}|${record.principalType}|${record.principalId}`, { ...record }); - }, - delete: async (contextGraphId: string, principalType: string, principalId: string) => { - persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); - }, - }; - const agentA = await DKGAgent.create({ - name: 'OnDemandSubscriptionLifetimeA', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const agentA = await createAgentWithContextGraphPersistence('OnDemandSubscriptionLifetimeA', fixture); try { await agentA.start(); @@ -213,13 +200,7 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => await agentA.stop().catch(() => {}); } - const agentB = await DKGAgent.create({ - name: 'OnDemandSubscriptionLifetimeB', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const agentB = await createAgentWithContextGraphPersistence('OnDemandSubscriptionLifetimeB', fixture); try { await agentB.start(); expect(agentB.getSubscribedContextGraphs().get('selected-cg')).toBeUndefined(); @@ -255,33 +236,10 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => it('promotes an existing on-demand selection when the node creates the graph', async () => { - const persisted = new Map(); - const persistedMembers = new Map(); - const subscriptionStore = { - loadAll: async () => [...persisted.values()], - save: async (record: any) => { - persisted.set(record.id, { ...record }); - }, - delete: async (contextGraphId: string) => { - persisted.delete(contextGraphId); - }, - }; - const membershipStore = { - upsert: async (record: any) => { - persistedMembers.set(`${record.contextGraphId}|${record.principalType}|${record.principalId}`, { ...record }); - }, - delete: async (contextGraphId: string, principalType: string, principalId: string) => { - persistedMembers.delete(`${contextGraphId}|${principalType}|${principalId}`); - }, - }; + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; const contextGraphId = 'selected-then-created-cg'; - const agent = await DKGAgent.create({ - name: 'OnDemandCreatePromotion', - listenHost: '127.0.0.1', - chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), - contextGraphSubscriptionStore: subscriptionStore, - contextGraphMembershipStore: membershipStore, - }); + const agent = await createAgentWithContextGraphPersistence('OnDemandCreatePromotion', fixture); try { await agent.start(); From 9cad49e921020493f14697cfca9582968f5eae9b Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 04:54:08 +0200 Subject: [PATCH 07/23] fix(cli): require explicit context graph sync mode --- packages/cli/src/api-client.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index 054d89bef..d271945b4 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -1490,9 +1490,9 @@ export class ApiClient { return this.post('/api/query-remote', { peerId, ...request }); } - async subscribeToContextGraph(contextGraphId: string, options?: { + async subscribeToContextGraph(contextGraphId: string, options: { includeSharedMemory?: boolean; - syncMode?: ContextGraphSyncMode; + syncMode: ContextGraphSyncMode; }): Promise<{ subscribed: string; syncMode: ContextGraphSyncMode; @@ -1559,8 +1559,8 @@ export class ApiClient { }> { return this.post('/api/context-graph/subscribe', { contextGraphId, - includeWorkspace: options?.includeSharedMemory, - syncMode: options?.syncMode, + includeWorkspace: options.includeSharedMemory, + syncMode: options.syncMode, }); } From 2b219d50299a95bbd3a3f0c9ca3a22a3bee617de Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:28:14 +0200 Subject: [PATCH 08/23] test(cli): cover legacy subscribe sync mode --- .../test/daemon-http-behavior-extra.test.ts | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index baee38636..8b02e4351 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -1247,6 +1247,118 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { } }); + it('applies syncMode through the legacy /api/subscribe alias and reports the agent-normalized mode', async () => { + const contextGraphId = 'legacy-subscribe-mode-' + Math.random().toString(36).slice(2, 8); + const catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + }; + let requestedMode: string | undefined; + const previousCatchupRunner = daemonState.catchupRunner; + daemonState.catchupRunner = { + run: async () => ({ + connectedPeers: 0, + syncCapablePeers: 0, + peersTried: 0, + peersResponded: 0, + peersSucceeded: 0, + deferredBackpressure: 1, + dataSynced: 0, + sharedMemorySynced: 0, + denied: false, + deniedPeers: 0, + }), + close: async () => {}, + } as any; + let routeServer: Server | null = null; + + try { + routeServer = createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + const agent = { + getContextGraphAllowedAgents: async () => [], + getSubscribedContextGraphs: () => new Map(), + subscribeToContextGraph: (_id: string, opts: { syncMode: string }) => { + requestedMode = opts.syncMode; + return { + subscribed: true, + synced: false, + // The route must report the normalized mode returned by the + // agent, rather than independently reconstructing it. + syncMode: 'always-on' as const, + }; + }, + markContextGraphSubscriptionState: () => {}, + resolveAgentByToken: () => undefined, + getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', + }; + + await handleContextGraphRoutes({ + req, + res, + agent, + publisherControl: {}, + publisherRuntime: null, + config: {}, + startedAt: Date.now(), + dashDb: {}, + opWallets: {}, + network: {}, + tracker: {}, + memoryManager: {}, + bridgeAuthToken: undefined, + nodeVersion: 'test', + nodeCommit: 'test', + catchupTracker, + extractionRegistry: {}, + fileStore: {}, + extractionStatus: new Map(), + assertionImportLocks: new Map(), + vectorStore: {}, + embeddingProvider: null, + validTokens: new Set(), + apiHost: '127.0.0.1', + apiPortRef: { value: 0 }, + routePlugins: [], + url, + path: url.pathname, + requestToken: undefined, + requestAgentAddress: '0x0000000000000000000000000000000000000001', + } as any); + if (!res.writableEnded) { + res.statusCode = 404; + res.end(); + } + }); + + await new Promise((resolve) => routeServer!.listen(0, '127.0.0.1', resolve)); + const address = routeServer.address(); + if (!address || typeof address === 'string') { + throw new Error('legacy subscribe route test server did not bind to a TCP port'); + } + + const response = await fetch(`http://127.0.0.1:${address.port}/api/subscribe`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ contextGraphId, syncMode: 'on-demand' }), + }); + expect(response.status).toBe(200); + expect(requestedMode).toBe('on-demand'); + expect(await response.json()).toMatchObject({ + subscribed: contextGraphId, + syncMode: 'always-on', + catchup: { status: 'queued' }, + }); + } finally { + daemonState.catchupRunner = previousCatchupRunner; + if (routeServer) { + await new Promise((resolve, reject) => { + routeServer!.close((err) => (err ? reject(err) : resolve())); + }); + } + } + }); + // SPEC_CG_MEMORY_MODEL / Codex PR #595 round-4: per-CG hosting // committees and per-CG quorum overrides were removed end-to-end. // The on-chain contract no longer accepts those args, so silently From 17b283026c63cded7fa479a352ec6ba70ebcf351 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:47:39 +0200 Subject: [PATCH 09/23] fix(agent): preserve dormant durable sync intent --- .../src/context-graph-subscription-policy.ts | 79 +++++++++++++++++ packages/agent/src/dkg-agent-cg-registry.ts | 2 + packages/agent/src/dkg-agent-context-graph.ts | 1 + packages/agent/src/dkg-agent-lifecycle.ts | 70 +++++++-------- packages/agent/src/dkg-agent-swm-host.ts | 13 ++- packages/agent/src/dkg-agent-swm-substrate.ts | 11 ++- packages/agent/src/dkg-agent-types.ts | 8 +- packages/agent/src/gossip-publish-handler.ts | 13 ++- packages/agent/test/agent.part-15.test.ts | 86 +++++++++++++++++++ 9 files changed, 224 insertions(+), 59 deletions(-) create mode 100644 packages/agent/src/context-graph-subscription-policy.ts diff --git a/packages/agent/src/context-graph-subscription-policy.ts b/packages/agent/src/context-graph-subscription-policy.ts new file mode 100644 index 000000000..25681b0e3 --- /dev/null +++ b/packages/agent/src/context-graph-subscription-policy.ts @@ -0,0 +1,79 @@ +import type { + ContextGraphSub, + ContextGraphSubInput, + ContextGraphSubscriptionRecord, + ContextGraphSyncMode, +} from './dkg-agent-types.js'; + +export function normalizeLegacyContextGraphSubscriptionInput( + previous: ContextGraphSub | undefined, + next: ContextGraphSubInput, +): ContextGraphSub { + return { + ...next, + syncMode: next.syncMode ?? previous?.syncMode ?? 'always-on', + }; +} + +export function resolveContextGraphSyncMode(input: { + existing?: Pick; + requested?: ContextGraphSyncMode; + hasDormantDurableIntent: boolean; +}): ContextGraphSyncMode { + if ( + input.hasDormantDurableIntent + || (input.existing?.subscribed === true && input.existing.syncMode === 'always-on') + ) { + return 'always-on'; + } + return input.requested ?? input.existing?.syncMode ?? 'always-on'; +} + +export type ContextGraphSubscriptionPersistenceProjection = + | { action: 'skip'; persistMemberIntent: false } + | { action: 'delete'; persistMemberIntent: true } + | { + action: 'save'; + persistMemberIntent: boolean; + record: ContextGraphSubscriptionRecord; + }; + +/** + * Canonical durable projection for live Context Graph subscription state. + * + * On-demand member intent remains process-local. A Core hosting obligation is + * independently durable and therefore projects to a host-only row. Always-on + * member intent projects the complete live readiness state. + */ +export function projectContextGraphSubscriptionPersistence(input: { + contextGraphId: string; + subscription: ContextGraphSub | undefined; + syncScoped: boolean; +}): ContextGraphSubscriptionPersistenceProjection { + const sub = input.subscription; + if (sub?.syncMode === 'on-demand' && sub.coreHosted !== true) { + return { action: 'skip', persistMemberIntent: false }; + } + if (!sub?.subscribed && !sub?.coreHosted) { + return { action: 'delete', persistMemberIntent: true }; + } + + const persistMemberIntent = sub.syncMode !== 'on-demand'; + return { + action: 'save', + persistMemberIntent, + record: { + id: input.contextGraphId, + name: sub.name, + subscribed: persistMemberIntent && sub.subscribed, + synced: persistMemberIntent && sub.synced, + sharedMemorySynced: persistMemberIntent ? sub.sharedMemorySynced : false, + metaSynced: persistMemberIntent ? sub.metaSynced : false, + onChainId: sub.onChainId, + onChainHash: sub.onChainHash, + lastReconciledOrdinal: sub.lastReconciledOrdinal, + coreHosted: sub.coreHosted, + syncScoped: persistMemberIntent && input.syncScoped, + }, + }; +} diff --git a/packages/agent/src/dkg-agent-cg-registry.ts b/packages/agent/src/dkg-agent-cg-registry.ts index d69d8c686..cd6a2c261 100644 --- a/packages/agent/src/dkg-agent-cg-registry.ts +++ b/packages/agent/src/dkg-agent-cg-registry.ts @@ -1051,6 +1051,7 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); this.setContextGraphSubscription(opts.id, { name: opts.name, + syncMode: 'always-on', subscribed: true, synced: true, metaSynced: true, @@ -1111,6 +1112,7 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { this.subscribeToContextGraph(opts.id, { syncMode: 'always-on' }); this.setContextGraphSubscription(opts.id, { name: opts.name, + syncMode: 'always-on', subscribed: true, synced: true, metaSynced: true, diff --git a/packages/agent/src/dkg-agent-context-graph.ts b/packages/agent/src/dkg-agent-context-graph.ts index 322e65ef9..d41a80477 100644 --- a/packages/agent/src/dkg-agent-context-graph.ts +++ b/packages/agent/src/dkg-agent-context-graph.ts @@ -696,6 +696,7 @@ export class ContextGraphMethods extends DKGAgentBase { this.setContextGraphSubscription(opts.id, { name: opts.name, + syncMode: 'always-on', subscribed: !opts.private, synced: true, metaSynced: true, diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index d4d8e8b1c..01e6d967e 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -457,7 +457,6 @@ import { type PeerDiagnostics, type ChatSendResult, type ContextGraphSub, - type ContextGraphSubInput, type ContextGraphSubscriptionRecord, type ContextGraphSubscriptionRehydrationStatus, type ContextGraphSubscriptionStore, @@ -475,6 +474,7 @@ import { type SyncReconcilerProbe, type SyncReconcilerBackoff, } from './dkg-agent-types.js'; +import { projectContextGraphSubscriptionPersistence } from './context-graph-subscription-policy.js'; import { authoritativeSyncPeerId, resolveCuratorSyncPeer, @@ -2339,6 +2339,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // upsert a minimal stub first. if (!this.subscribedContextGraphs.has(localId)) { this.setContextGraphSubscription(localId, { + syncMode: 'always-on', subscribed: false, synced: false, onChainHash: hashLower, @@ -6499,7 +6500,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { setContextGraphSubscription(this: DKGAgent, contextGraphId: string, - next: ContextGraphSubInput, + next: ContextGraphSub, options?: { persist?: boolean; updateRehydrationStatus?: boolean }, ): ContextGraphSub { this.invalidateListContextGraphsCache(); @@ -6517,7 +6518,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { const nextWireId = nextOnChainHash ?? localWireId; const canonicalNext: ContextGraphSub = { ...next, - syncMode: next.syncMode ?? previous?.syncMode ?? 'always-on', ...(next.onChainHash === nextOnChainHash ? {} : { onChainHash: nextOnChainHash }), }; if ( @@ -6535,10 +6535,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { // readiness process-local. A Core's independent hosting obligation is // still durable, though: persistContextGraphSubscription writes a // host-only snapshot without converting the member intent to always-on. - if ( - options?.persist !== false && - (canonicalNext.syncMode !== 'on-demand' || canonicalNext.coreHosted === true) - ) { + const persistence = projectContextGraphSubscriptionPersistence({ + contextGraphId, + subscription: canonicalNext, + syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), + }); + if (options?.persist !== false && persistence.action !== 'skip') { if (this.config.contextGraphSubscriptionStore) { const revision = this.nextContextGraphSubscriptionPersistRevision(contextGraphId); this.persistContextGraphSubscription( @@ -6549,9 +6551,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { }, ); } - if (canonicalNext.syncMode !== 'on-demand' && canonicalNext.subscribed) { + if (persistence.persistMemberIntent && canonicalNext.subscribed) { this.persistLocalNodeMembership(contextGraphId); - } else if (canonicalNext.syncMode !== 'on-demand') { + } else if (persistence.persistMemberIntent) { this.deleteContextGraphMember(contextGraphId, 'node', this.peerId); } } @@ -6774,7 +6776,12 @@ export class LifecycleSyncMethods extends DKGAgentBase { return; } const sub = this.subscribedContextGraphs.get(contextGraphId); - if (sub?.syncMode === 'on-demand' && sub.coreHosted !== true) { + const persistence = projectContextGraphSubscriptionPersistence({ + contextGraphId, + subscription: sub, + syncScoped: (this.config.syncContextGraphs ?? []).includes(contextGraphId), + }); + if (persistence.action === 'skip') { // Some lifecycle paths persist reconciliation watermarks directly // instead of going through setContextGraphSubscription. Preserve the // process-local lifetime at this lowest shared write boundary too. @@ -6786,7 +6793,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // during a publish remembers it hosts the CG and fills its gap. Drop the // row only when the node neither subscribes to nor hosts the CG. this.beginContextGraphSubscriptionPersistRevision(contextGraphId, options?.revision); - if (!sub?.subscribed && !sub?.coreHosted) { + if (persistence.action === 'delete') { void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.delete(contextGraphId)) .then(() => { if ( @@ -6807,24 +6814,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { }); return; } - const persistMemberIntent = sub.syncMode !== 'on-demand'; - // Project the durable record directly from the real live state. An - // on-demand member can coexist with an independent Core-hosting duty, but - // that host-only row must not masquerade as a second live subscription. - const record: ContextGraphSubscriptionRecord = { - id: contextGraphId, - name: sub.name, - subscribed: persistMemberIntent && sub.subscribed, - synced: persistMemberIntent && sub.synced, - sharedMemorySynced: persistMemberIntent ? sub.sharedMemorySynced : false, - metaSynced: persistMemberIntent ? sub.metaSynced : false, - onChainId: sub.onChainId, - onChainHash: sub.onChainHash, - lastReconciledOrdinal: sub.lastReconciledOrdinal, - coreHosted: sub.coreHosted, - syncScoped: - persistMemberIntent && (this.config.syncContextGraphs ?? []).includes(contextGraphId), - }; + const record = persistence.record; void this.enqueueContextGraphSubscriptionPersistWrite(contextGraphId, () => store.save(record)) .then(() => { if ( @@ -6861,19 +6851,17 @@ export class LifecycleSyncMethods extends DKGAgentBase { `Cannot acknowledge join approval for "${contextGraphId}": active subscription state is missing`, ); } - const record = { - id: contextGraphId, - name: sub.name, - subscribed: sub.subscribed, - synced: sub.synced, - sharedMemorySynced: sub.sharedMemorySynced, - metaSynced: sub.metaSynced, - onChainId: sub.onChainId, - onChainHash: sub.onChainHash, - lastReconciledOrdinal: sub.lastReconciledOrdinal, - coreHosted: sub.coreHosted, + const persistence = projectContextGraphSubscriptionPersistence({ + contextGraphId, + subscription: sub, syncScoped: syncScoped ?? (this.config.syncContextGraphs ?? []).includes(contextGraphId), - }; + }); + if (persistence.action !== 'save' || !persistence.persistMemberIntent) { + throw new Error( + `Cannot acknowledge join approval for "${contextGraphId}": durable subscription intent is missing`, + ); + } + const record = persistence.record; // Queue behind any fire-and-forget writes scheduled by subscribe/mark so // this final authoritative snapshot is the last write before the ACK. await this.enqueueContextGraphSubscriptionPersistWrite( diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index 5dc545df2..854797a22 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -352,7 +352,6 @@ import { type PeerDiagnostics, type ChatSendResult, type ContextGraphSub, - type ContextGraphSubInput, type ContextGraphSubscriptionRecord, type ContextGraphSubscriptionStore, type ContextGraphMemberPrincipalType, @@ -1129,6 +1128,7 @@ export class SwmHostModeMethods extends DKGAgentBase { // translate either direction without an extra RPC. if (storageCgId !== contextGraphId) { const storageSubscription = this.subscribedContextGraphs.get(storageCgId) ?? { + syncMode: 'always-on' as const, subscribed: false, synced: false, pendingMeta: true, @@ -1697,6 +1697,7 @@ export class SwmHostModeMethods extends DKGAgentBase { // that did not create or join the CG. if (!this.subscribedContextGraphs.has(wireId)) { this.setContextGraphSubscription(wireId, { + syncMode: 'always-on', subscribed: false, synced: false, onChainHash: wireId, @@ -2438,7 +2439,7 @@ export class SwmHostModeMethods extends DKGAgentBase { const existing = this.subscribedContextGraphs.get(localCgId); if (existing?.coreHosted && existing.onChainId === numericStr) return; // already recorded - let next: ContextGraphSubInput; + let next: ContextGraphSub; if (existing) { // Rebind through the helper so a CG re-created/rebound under the same // local id drops its stale reconcile watermark + in-memory cursor before @@ -2449,7 +2450,13 @@ export class SwmHostModeMethods extends DKGAgentBase { existing.coreHosted = true; next = existing; } else { - next = { subscribed: false, synced: false, onChainId: numericStr, coreHosted: true }; + next = { + syncMode: 'always-on', + subscribed: false, + synced: false, + onChainId: numericStr, + coreHosted: true, + }; } this.setContextGraphSubscription(localCgId, next); this.log.info( diff --git a/packages/agent/src/dkg-agent-swm-substrate.ts b/packages/agent/src/dkg-agent-swm-substrate.ts index 29fdbc114..c13b8f8d0 100644 --- a/packages/agent/src/dkg-agent-swm-substrate.ts +++ b/packages/agent/src/dkg-agent-swm-substrate.ts @@ -349,6 +349,7 @@ import { type DKGAgentConfig, type ReplicationEvent, } from './dkg-agent-types.js'; +import { resolveContextGraphSyncMode } from './context-graph-subscription-policy.js'; import { normalizePublishContextGraphId, isPublishAsyncQuadEnvelope, @@ -399,10 +400,12 @@ export class SwmSubstrateMethods extends DKGAgentBase { // process-local subscription. An explicit always-on request may promote an // existing on-demand subscription, while an omitted mode preserves the // current lifetime (or the legacy always-on default for a new graph). - const requestedSyncMode = options?.syncMode ?? existing?.syncMode ?? 'always-on'; - const syncMode = existing?.subscribed && existing.syncMode === 'always-on' - ? 'always-on' - : requestedSyncMode; + const syncMode = resolveContextGraphSyncMode({ + existing, + requested: options?.syncMode, + hasDormantDurableIntent: + this.contextGraphSubscriptionRehydrationStatus?.dormantIds.includes(contextGraphId) === true, + }); const persist = syncMode === 'on-demand' ? false : options?.persist; // SWM gossip subscribe runs `canReadContextGraph` against the local diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index 090f97e80..d6accdf49 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -698,11 +698,11 @@ export interface ContextGraphSub { } /** - * Compatibility input accepted at subscription-update boundaries. + * Legacy compatibility input for the standalone gossip handler only. * - * Historical call sites and persisted records predate synchronization modes, - * so they may omit the field. The agent normalizes this shape exactly once - * before storing it in the live {@link ContextGraphSub} registry. + * Agent-owned live-state mutations use normalized {@link ContextGraphSub} + * values and must choose a synchronization lifetime explicitly. This shape is + * retained solely for older standalone handler callbacks that predate modes. */ export type ContextGraphSubInput = Omit & { syncMode?: ContextGraphSyncMode; diff --git a/packages/agent/src/gossip-publish-handler.ts b/packages/agent/src/gossip-publish-handler.ts index 4038f7e29..9a7c5c958 100644 --- a/packages/agent/src/gossip-publish-handler.ts +++ b/packages/agent/src/gossip-publish-handler.ts @@ -36,6 +36,7 @@ import type { ContextGraphSub, ContextGraphSubInput, } from './dkg-agent-types.js'; +import { normalizeLegacyContextGraphSubscriptionInput } from './context-graph-subscription-policy.js'; import { protobufScalarToBigInt, protobufScalarToNumber } from './protobuf-scalars.js'; export type GossipPhaseCallback = (phase: string, status: 'start' | 'end') => void; @@ -136,7 +137,7 @@ function resolveGraphScopedPublishRequest( export interface GossipPublishHandlerCallbacks { contextGraphExists: (id: string) => Promise; getContextGraphOwner: (id: string) => Promise; - setContextGraphSubscription?: (id: string, next: ContextGraphSubInput, options?: { persist?: boolean }) => void; + setContextGraphSubscription?: (id: string, next: ContextGraphSub, options?: { persist?: boolean }) => void; /** * Record a Context Graph learned from ontology gossip. Agent-backed callers * apply the node-role policy centrally (edge=catalogue-only, core=activate); @@ -202,16 +203,14 @@ export class GossipPublishHandler { next: ContextGraphSubInput, options?: { persist?: boolean }, ): void { + const previous = this.subscribedContextGraphs.get(id); + const normalized = normalizeLegacyContextGraphSubscriptionInput(previous, next); const setter = this.callbacks.setContextGraphSubscription; if (setter) { - setter(id, next, options); + setter(id, normalized, options); return; } - const previous = this.subscribedContextGraphs.get(id); - this.subscribedContextGraphs.set(id, { - ...next, - syncMode: next.syncMode ?? previous?.syncMode ?? 'always-on', - }); + this.subscribedContextGraphs.set(id, normalized); } private recordDiscoveredContextGraph(id: string, metadata: ContextGraphDiscoveryMetadata): void { diff --git a/packages/agent/test/agent.part-15.test.ts b/packages/agent/test/agent.part-15.test.ts index 9885a7801..e5b3f2369 100644 --- a/packages/agent/test/agent.part-15.test.ts +++ b/packages/agent/test/agent.part-15.test.ts @@ -235,6 +235,92 @@ describe('DKGAgent config — syncContextGraphs and queryAccess warning', () => }); + it('preserves dormant durable intent when an edge opens the graph on demand', async () => { + const fixture = createContextGraphPersistenceFixture(); + const { persisted, persistedMembers } = fixture; + const activeId = 'durable-a-active'; + const dormantId = 'durable-z-dormant'; + for (const id of [activeId, dormantId]) { + persisted.set(id, { + id, + name: id, + subscribed: true, + synced: false, + sharedMemorySynced: false, + metaSynced: false, + syncScoped: true, + }); + } + + const agentA = await DKGAgent.create({ + name: 'DormantDurableIntentA', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: fixture.subscriptionStore, + contextGraphMembershipStore: fixture.membershipStore, + maxRehydratedContextGraphSubscriptions: 1, + }); + + try { + await agentA.start(); + expect(agentA.getSubscribedContextGraphs().get(activeId)?.syncMode).toBe('always-on'); + expect(agentA.getSubscribedContextGraphs().get(dormantId)).toBeUndefined(); + expect(agentA.getContextGraphSubscriptionRehydrationStatus()).toMatchObject({ + activated: 1, + dormant: 1, + dormantIds: [dormantId], + }); + + // Opening a capped durable row is expressed by the UI as on-demand. + // The agent must recover the stored always-on intent instead of + // downgrading it and later deleting it from the durable store. + const selected = agentA.subscribeToContextGraph(dormantId, { syncMode: 'on-demand' }); + expect(selected.syncMode).toBe('always-on'); + expect(agentA.getSubscribedContextGraphs().get(dormantId)).toMatchObject({ + subscribed: true, + syncMode: 'always-on', + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(persisted.get(dormantId)).toMatchObject({ + id: dormantId, + subscribed: true, + syncScoped: true, + }); + expect(persistedMembers.get(`${dormantId}|node|${agentA.peerId}`)).toMatchObject({ + contextGraphId: dormantId, + status: 'active', + source: 'subscription', + }); + expect(agentA.getContextGraphSubscriptionRehydrationStatus()).toMatchObject({ + activated: 2, + dormant: 0, + dormantIds: [], + }); + } finally { + await agentA.stop().catch(() => {}); + } + + const agentB = await DKGAgent.create({ + name: 'DormantDurableIntentB', + listenHost: '127.0.0.1', + chainAdapter: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + contextGraphSubscriptionStore: fixture.subscriptionStore, + contextGraphMembershipStore: fixture.membershipStore, + maxRehydratedContextGraphSubscriptions: 0, + }); + try { + await agentB.start(); + expect(agentB.getSubscribedContextGraphs().get(dormantId)).toMatchObject({ + subscribed: true, + syncMode: 'always-on', + }); + } finally { + await agentB.stop().catch(() => {}); + } + }); + + it('promotes an existing on-demand selection when the node creates the graph', async () => { const fixture = createContextGraphPersistenceFixture(); const { persisted, persistedMembers } = fixture; From 9471694c55f318f65f7691983bbde4fe5bfddb46 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:47:42 +0200 Subject: [PATCH 10/23] fix(cli): reject null context graph sync mode --- .../cli/src/daemon/routes/context-graph.ts | 6 ++- .../test/daemon-http-behavior-extra.test.ts | 42 +++++++++++++------ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 89031e04d..2fef646df 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -1685,7 +1685,11 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise { latestByContextGraph: new Map(), }; let requestedMode: string | undefined; + let runnerCalls = 0; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { - run: async () => ({ - connectedPeers: 0, - syncCapablePeers: 0, - peersTried: 0, - peersResponded: 0, - peersSucceeded: 0, - deferredBackpressure: 1, - dataSynced: 0, - sharedMemorySynced: 0, - denied: false, - deniedPeers: 0, - }), + run: async () => { + runnerCalls += 1; + return { + connectedPeers: 0, + syncCapablePeers: 0, + peersTried: 0, + peersResponded: 0, + peersSucceeded: 0, + deferredBackpressure: 1, + dataSynced: 0, + sharedMemorySynced: 0, + denied: false, + deniedPeers: 0, + }; + }, close: async () => {}, } as any; let routeServer: Server | null = null; @@ -1337,6 +1341,20 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { throw new Error('legacy subscribe route test server did not bind to a TCP port'); } + const invalidResponse = await fetch(`http://127.0.0.1:${address.port}/api/subscribe`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ contextGraphId, syncMode: null }), + }); + expect(invalidResponse.status).toBe(400); + expect(await invalidResponse.json()).toMatchObject({ + error: 'Invalid "syncMode" (expected "on-demand" or "always-on")', + }); + expect(requestedMode).toBeUndefined(); + expect(runnerCalls).toBe(0); + expect(catchupTracker.jobs.size).toBe(0); + expect(catchupTracker.latestByContextGraph.size).toBe(0); + const response = await fetch(`http://127.0.0.1:${address.port}/api/subscribe`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, From 42a0e6636425ae4688d75e13013e4b4e8ad764a4 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 03:05:07 +0200 Subject: [PATCH 11/23] feat(sync): expose verified CG convergence by plane --- packages/cli/src/api-client.ts | 21 +++ packages/cli/src/cli-helpers.ts | 25 +++ packages/cli/src/context-graph-readiness.ts | 105 +++++++++--- .../cli/src/daemon/routes/context-graph.ts | 155 +++++++++++------- packages/cli/src/daemon/routes/query.ts | 19 ++- packages/cli/src/daemon/types.ts | 25 ++- .../cli/test/catchup-status-response.test.ts | 60 +++++++ .../context-graph-catchup-readiness.test.ts | 78 +++++++++ .../context-graph-subscribe-readiness.test.ts | 149 ++++++++++++++++- 9 files changed, 546 insertions(+), 91 deletions(-) create mode 100644 packages/cli/test/catchup-status-response.test.ts diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index d271945b4..21d6e034a 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -1648,6 +1648,7 @@ export class ApiClient { contextGraphId: string; includeWorkspace: boolean; status: 'queued' | 'running' | 'done' | 'denied' | 'deferred' | 'failed' | 'unreachable'; + attemptStatus: 'queued' | 'running' | 'done' | 'denied' | 'deferred' | 'failed' | 'unreachable'; queuedAt: number; startedAt?: number; finishedAt?: number; @@ -1706,6 +1707,26 @@ export class ApiClient { }; }; error?: string; + attemptError?: string; + convergence?: { + state: 'pending' | 'partial' | 'complete'; + required: { + metadata: true; + durable: true; + sharedMemory: boolean; + }; + verified: { + metadata: boolean; + durable: boolean; + sharedMemory: boolean; + }; + missing: Array<'metadata' | 'durable' | 'sharedMemory'>; + readinessUpdatedAt?: number; + observedAt: number; + syncMode: 'on-demand' | 'always-on'; + automaticRetryActive: boolean; + }; + completedAfterAttempt?: true; }> { return this.get(`/api/sync/catchup-status?contextGraphId=${encodeURIComponent(contextGraphId)}`); } diff --git a/packages/cli/src/cli-helpers.ts b/packages/cli/src/cli-helpers.ts index d572fd554..8f307241c 100644 --- a/packages/cli/src/cli-helpers.ts +++ b/packages/cli/src/cli-helpers.ts @@ -202,6 +202,10 @@ function printCatchupStatus(status: Awaited 0) { + console.log(`Missing: ${status.convergence.missing.join(', ')}`); + } + console.log( + `Retry: ${status.convergence.automaticRetryActive ? 'active' : 'inactive'} ` + + `(${status.convergence.syncMode})`, + ); + if (status.completedAfterAttempt) { + console.log('Recovered: a later synchronization completed the selected graph'); + } + } if ( status.result && (status.result.selectedPeers ?? status.result.connectedPeers) >= (status.result.totalPeers ?? status.result.connectedPeers) && diff --git a/packages/cli/src/context-graph-readiness.ts b/packages/cli/src/context-graph-readiness.ts index 91aab03a6..6794681e9 100644 --- a/packages/cli/src/context-graph-readiness.ts +++ b/packages/cli/src/context-graph-readiness.ts @@ -42,6 +42,73 @@ export interface ContextGraphReadinessPatch { sharedMemoryVerified: boolean; } +export type ContextGraphConvergencePlane = 'metadata' | 'durable' | 'sharedMemory'; + +export interface ContextGraphConvergenceSnapshot { + state: 'pending' | 'partial' | 'complete'; + required: { + metadata: true; + durable: true; + sharedMemory: boolean; + }; + verified: { + metadata: boolean; + durable: boolean; + sharedMemory: boolean; + }; + missing: ContextGraphConvergencePlane[]; + readinessUpdatedAt?: number; + observedAt: number; +} + +/** + * Describe the live, per-plane convergence of one selected context graph. + * A VM/SWM proof is only effective while authoritative CG metadata is local; + * stale provenance must not make a graph look complete after metadata loss. + */ +export function describeContextGraphConvergence(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; + observedAt?: number; +}): ContextGraphConvergenceSnapshot { + const currentReadinessProvenance = + input.readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; + const metadataVerified = input.hasConfirmedMeta; + const durableVerified = metadataVerified && + currentReadinessProvenance && + input.readiness.durableVerified; + const sharedMemoryVerified = metadataVerified && + currentReadinessProvenance && + input.readiness.sharedMemoryVerified; + const missing: ContextGraphConvergencePlane[] = []; + if (!metadataVerified) missing.push('metadata'); + if (!durableVerified) missing.push('durable'); + if (input.includeSharedMemory && !sharedMemoryVerified) missing.push('sharedMemory'); + + const anyVerified = metadataVerified || durableVerified || + (input.includeSharedMemory && sharedMemoryVerified); + + return { + state: missing.length === 0 ? 'complete' : anyVerified ? 'partial' : 'pending', + required: { + metadata: true, + durable: true, + sharedMemory: input.includeSharedMemory, + }, + verified: { + metadata: metadataVerified, + durable: durableVerified, + sharedMemory: sharedMemoryVerified, + }, + missing, + ...(currentReadinessProvenance + ? { readinessUpdatedAt: input.readiness.updatedAt } + : {}), + observedAt: input.observedAt ?? Date.now(), + }; +} + export interface MissingMetadataReadinessPatches { statePatch: ContextGraphSubscriptionStatePatch; readinessPatch: ContextGraphReadinessPatch; @@ -75,11 +142,9 @@ export function classifyExistingContextGraphReadiness(input: { } { const currentReadinessProvenance = input.readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; - const overallReadinessVerified = - input.readiness.durableVerified || input.readiness.sharedMemoryVerified; const requestedPlanesVerified = currentReadinessProvenance && - overallReadinessVerified && + input.readiness.durableVerified && (!input.includeSharedMemory || input.readiness.sharedMemoryVerified); const alreadyReady = input.hasConfirmedMeta && @@ -113,12 +178,11 @@ export function classifyExistingContextGraphReadiness(input: { currentReadinessProvenance && input.readiness.durableVerified; const sharedMemoryVerified = currentReadinessProvenance && input.readiness.sharedMemoryVerified; - const overallVerified = durableVerified || sharedMemoryVerified; const statePatch = - input.subscription.synced !== overallVerified || + input.subscription.synced !== durableVerified || input.subscription.sharedMemorySynced !== sharedMemoryVerified ? { - synced: overallVerified, + synced: durableVerified, sharedMemorySynced: sharedMemoryVerified, metaSynced: true, pendingMeta: false, @@ -351,17 +415,7 @@ export function classifyContextGraphCatchupReadiness(input: { const durableVerifiedPersisted = durableVerifiedBefore || durableThisRun.persistable; const sharedMemoryVerifiedPersisted = sharedMemoryVerifiedBefore || sharedMemoryThisRun.persistable; - // `subscription.synced` is a SECOND persisted readiness bit, living outside - // the provenance store and consumed by callers that never see this job's - // result — `contextGraphRowIsWritable` treats `subscribed && synced` as - // writable. It must therefore carry the same verdict as `readinessPatch`, - // not the transient one. The pre-catch-up path already assumes they agree: - // it derives `synced` from the persisted provenance and patches the row - // back into line, so letting them diverge here would be corrected away on - // the next pass anyway — after a window in which the graph looked writable. - const overallVerifiedPersisted = durableVerifiedPersisted || sharedMemoryVerifiedPersisted; - const overallVerified = durableVerified || sharedMemoryVerified; - const missingGraphProof = !overallVerified; + const missingDurable = !durableVerified; const missingRequestedSharedMemory = input.includeSharedMemory && !sharedMemoryVerified; const madeIncompleteProgress = @@ -370,14 +424,18 @@ export function classifyContextGraphCatchupReadiness(input: { let jobStatus: ContextGraphCatchupReadinessClassification['jobStatus'] = 'done'; let error: string | undefined; - if (missingGraphProof || missingRequestedSharedMemory) { + if (missingDurable || missingRequestedSharedMemory) { jobStatus = 'unreachable'; if (madeIncompleteProgress) { error = 'Verified data was inserted, but catch-up did not complete without a timeout or failed phase. The incomplete plane remains unready; retry once the network is healthier.'; - } else if (input.isPrivate && missingGraphProof) { - error = 'No authorized context-graph peer delivered verified durable or shared-memory data — empty or metadata-only responses cannot prove a private graph is fully synchronized, and the curator may be offline.'; - } else if (input.isPrivate) { + } else if (input.isPrivate && missingDurable && sharedMemoryVerified) { + error = 'Shared-memory catch-up completed, but no authorized peer delivered a verified durable VM snapshot. The selected graph remains incomplete.'; + } else if (input.isPrivate && missingDurable) { + error = 'No authorized context-graph peer delivered verified durable VM data — empty or metadata-only responses cannot prove a private graph is fully synchronized, and the curator may be offline.'; + } else if (input.isPrivate && missingRequestedSharedMemory) { error = 'Durable context-graph data synchronized, but shared-memory catch-up did not complete. Retry to finish shared-memory synchronization.'; + } else if (missingDurable && sharedMemoryVerified) { + error = 'Shared-memory catch-up completed, but durable VM catch-up did not complete. The selected graph remains incomplete.'; } else { error = 'Context-graph catch-up did not complete cleanly for every requested data plane. Retry once the network is healthier.'; } @@ -387,7 +445,10 @@ export function classifyContextGraphCatchupReadiness(input: { jobStatus, error, statePatch: { - synced: overallVerifiedPersisted, + // `synced` is a second persisted VM-readiness bit used by write + // preflight, so it must match durable provenance rather than transient + // empty-round evidence or independently verified SWM. + synced: durableVerifiedPersisted, sharedMemorySynced: sharedMemoryVerifiedPersisted, metaSynced: true, pendingMeta: false, diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 2fef646df..f05dc6a95 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -1747,6 +1747,12 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise 0 && !result.denied) { - job.status = "deferred"; - job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} deferred by local scheduler: ${result.deferredBackpressure}`); - } else { - const inspectReadiness = catchupResultHasCleanResponse(result); - const hasConfirmedMeta = inspectReadiness - ? await agent.hasConfirmedMetaState(contextGraphId).catch(() => false) - : false; - const isPrivate = hasConfirmedMeta - ? await agent.isPrivateContextGraph(contextGraphId).catch(() => true) - : false; - const classification = classifyContextGraphCatchupReadiness({ - result, - includeSharedMemory: shouldSyncSharedMemory, - hasConfirmedMeta, - isPrivate, - readinessBeforeCatchup, + let attemptReadinessBeforeCatchup = readinessBeforeCatchup; + while (true) { + const attemptIncludeSharedMemory = job.includeWorkspace; + const result = await daemonState.catchupRunner!.run({ + contextGraphId: contextGraphId, + includeSharedMemory: attemptIncludeSharedMemory, }); - - job.status = classification.jobStatus; - job.error = classification.error; - if (classification.readinessPatch) { - writeContextGraphReadiness( - dashDb, - contextGraphId, - classification.readinessPatch, - ); - } - if (classification.statePatch) { - agent.markContextGraphSubscriptionState( - contextGraphId, - classification.statePatch, - ); - } - if (classification.eventPayload) { - agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { - contextGraphId, - ...classification.eventPayload, + job.result = result; + job.error = undefined; + // Local scheduler pressure cut the round short. An incomplete round has + // no readiness to inspect and must never finalize the subscription, so + // short-circuit the whole classification path and report a distinct + // retryable status. A remote denial still wins: waiting for local + // capacity will never clear it. + if (result.deferredBackpressure > 0 && !result.denied) { + job.status = "deferred"; + job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; + if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} deferred by local scheduler: ${result.deferredBackpressure}`); + } else { + const inspectReadiness = catchupResultHasCleanResponse(result); + const hasConfirmedMeta = inspectReadiness + ? await agent.hasConfirmedMetaState(contextGraphId).catch(() => false) + : false; + const isPrivate = hasConfirmedMeta + ? await agent.isPrivateContextGraph(contextGraphId).catch(() => true) + : false; + const classification = classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory: attemptIncludeSharedMemory, + hasConfirmedMeta, + isPrivate, + readinessBeforeCatchup: attemptReadinessBeforeCatchup, }); + + job.status = classification.jobStatus; + job.error = classification.error; + if (classification.readinessPatch) { + writeContextGraphReadiness( + dashDb, + contextGraphId, + classification.readinessPatch, + ); + } + if (classification.statePatch) { + agent.markContextGraphSubscriptionState( + contextGraphId, + classification.statePatch, + ); + } + if (classification.eventPayload) { + agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { + contextGraphId, + ...classification.eventPayload, + }); + } + + // Denial took precedence above, but a denied-yet-still-deferred round + // is likewise incomplete: never let it settle as a successful "done". + if (job.status === "done" && result.deferredBackpressure > 0) { + job.status = "deferred"; + job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; + } } - // Denial took precedence above, but a denied-yet-still-deferred round - // is likewise incomplete: never let it settle as a successful "done". - if (job.status === "done" && result.deferredBackpressure > 0) { - job.status = "deferred"; - job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; + if (DEBUG_SYNC_TRACE) { + if (job.status === 'denied') { + console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} denied by remote peer(s): ${result.deniedPeers}`); + } + console.log( + `[catchup] job=${jobId} contextGraph=${contextGraphId} status=${job.status} ` + + `peers=${result.peersTried}/${result.syncCapablePeers} ` + + `connected=${result.totalPeers ?? result.connectedPeers} ` + + `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, + ); } - } - if (DEBUG_SYNC_TRACE) { - if (job.status === 'denied') { - console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} denied by remote peer(s): ${result.deniedPeers}`); + // A caller may upgrade a running VM-only selection to VM+SWM. Finish + // the narrow attempt, then satisfy the wider request in the same job + // instead of launching a competing per-CG catch-up. + if ( + job.includeWorkspace && + !attemptIncludeSharedMemory && + job.status !== 'denied' + ) { + job.status = 'running'; + job.error = undefined; + attemptReadinessBeforeCatchup = readContextGraphReadiness( + dashDb, + contextGraphId, + ); + continue; } - console.log( - `[catchup] job=${jobId} contextGraph=${contextGraphId} status=${job.status} ` + - `peers=${result.peersTried}/${result.syncCapablePeers} ` + - `connected=${result.totalPeers ?? result.connectedPeers} ` + - `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, - ); + break; } } catch (err) { job.error = err instanceof Error ? err.message : String(err); diff --git a/packages/cli/src/daemon/routes/query.ts b/packages/cli/src/daemon/routes/query.ts index e4d14e3e9..7881d169d 100644 --- a/packages/cli/src/daemon/routes/query.ts +++ b/packages/cli/src/daemon/routes/query.ts @@ -118,6 +118,10 @@ export { export type { ApiQueryPriority } from '../api-query-priority.js'; import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; +import { + describeContextGraphConvergence, + readContextGraphReadiness, +} from '../../context-graph-readiness.js'; import { loadTokens, httpAuthGuard, extractBearerToken } from '../../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; import { MarkItDownConverter, isMarkItDownAvailable, extractFromMarkdown, extractWithLlm } from '../../extraction/index.js'; @@ -1033,7 +1037,20 @@ export async function handleQueryRoutes(ctx: RequestContext): Promise { }); } - return jsonResponse(res, 200, toCatchupStatusResponse(job)); + const subscription = agent.getSubscribedContextGraphs().get(job.contextGraphId); + const hasConfirmedMeta = await agent.hasConfirmedMetaState(job.contextGraphId) + .catch(() => false); + const convergence = { + ...describeContextGraphConvergence({ + readiness: readContextGraphReadiness(dashDb, job.contextGraphId), + includeSharedMemory: job.includeWorkspace, + hasConfirmedMeta, + }), + syncMode: subscription?.syncMode ?? 'always-on', + automaticRetryActive: subscription?.subscribed === true, + }; + + return jsonResponse(res, 200, toCatchupStatusResponse(job, convergence)); } // POST /api/verify diff --git a/packages/cli/src/daemon/types.ts b/packages/cli/src/daemon/types.ts index 2da37e8b5..f215bde3c 100644 --- a/packages/cli/src/daemon/types.ts +++ b/packages/cli/src/daemon/types.ts @@ -3,6 +3,7 @@ // Pure type/interface declarations used across the daemon sub-modules. import type { CatchupJobResult } from '../catchup-runner.js'; +import type { ContextGraphConvergenceSnapshot } from '../context-graph-readiness.js'; export type CatchupJobState = | "queued" @@ -39,10 +40,32 @@ export interface CatchupTracker { latestByContextGraph: Map; } -export function toCatchupStatusResponse(job: CatchupJob) { +export interface CatchupConvergenceStatus extends ContextGraphConvergenceSnapshot { + syncMode: 'on-demand' | 'always-on'; + automaticRetryActive: boolean; +} + +export function toCatchupStatusResponse( + job: CatchupJob, + convergence?: CatchupConvergenceStatus, +) { + const completedAfterAttempt = convergence?.state === 'complete' && + (job.status === 'failed' || + job.status === 'deferred' || + job.status === 'unreachable'); return { ...job, contextGraphId: job.contextGraphId, includeSharedMemory: job.includeWorkspace, + attemptStatus: job.status, + ...(completedAfterAttempt + ? { + status: 'done' as const, + error: undefined, + ...(job.error ? { attemptError: job.error } : {}), + } + : {}), + ...(convergence ? { convergence } : {}), + ...(completedAfterAttempt ? { completedAfterAttempt: true } : {}), }; } diff --git a/packages/cli/test/catchup-status-response.test.ts b/packages/cli/test/catchup-status-response.test.ts new file mode 100644 index 000000000..0e4953dfd --- /dev/null +++ b/packages/cli/test/catchup-status-response.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { + toCatchupStatusResponse, + type CatchupConvergenceStatus, + type CatchupJob, +} from '../src/daemon/types.js'; + +const completeConvergence: CatchupConvergenceStatus = { + state: 'complete', + required: { + metadata: true, + durable: true, + sharedMemory: true, + }, + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + readinessUpdatedAt: 10, + observedAt: 20, + syncMode: 'on-demand', + automaticRetryActive: true, +}; + +function job(status: CatchupJob['status']): CatchupJob { + return { + jobId: 'job-1', + contextGraphId: 'cg-1', + includeWorkspace: true, + status, + queuedAt: 1, + startedAt: 2, + finishedAt: 3, + error: `${status} attempt`, + }; +} + +describe('catch-up status response', () => { + it('reports live completion while preserving a failed attempt as diagnostics', () => { + expect(toCatchupStatusResponse(job('failed'), completeConvergence)).toMatchObject({ + status: 'done', + attemptStatus: 'failed', + attemptError: 'failed attempt', + error: undefined, + completedAfterAttempt: true, + convergence: completeConvergence, + }); + }); + + it('never lets historical readiness override a current authorization denial', () => { + expect(toCatchupStatusResponse(job('denied'), completeConvergence)).toMatchObject({ + status: 'denied', + attemptStatus: 'denied', + error: 'denied attempt', + convergence: completeConvergence, + }); + }); +}); diff --git a/packages/cli/test/context-graph-catchup-readiness.test.ts b/packages/cli/test/context-graph-catchup-readiness.test.ts index 807190133..9d27875c4 100644 --- a/packages/cli/test/context-graph-catchup-readiness.test.ts +++ b/packages/cli/test/context-graph-catchup-readiness.test.ts @@ -3,6 +3,7 @@ import type { CatchupJobResult } from '../src/catchup-runner.js'; import { CONTEXT_GRAPH_READINESS_VERSION, classifyContextGraphCatchupReadiness, + describeContextGraphConvergence, } from '../src/context-graph-readiness.js'; function mixedPeerResult(verifiedDataPeers: number): CatchupJobResult { @@ -531,3 +532,80 @@ describe('context graph catch-up readiness classification', () => { }).jobStatus).toBe('unreachable'); }); }); + +describe('selected context-graph convergence snapshot', () => { + it('requires independently verified VM and requested SWM planes', () => { + const snapshot = describeContextGraphConvergence({ + readiness: { + version: 1, + durableVerified: false, + sharedMemoryVerified: true, + updatedAt: 42, + }, + includeSharedMemory: true, + hasConfirmedMeta: true, + observedAt: 100, + }); + + expect(snapshot).toEqual({ + state: 'partial', + required: { + metadata: true, + durable: true, + sharedMemory: true, + }, + verified: { + metadata: true, + durable: false, + sharedMemory: true, + }, + missing: ['durable'], + readinessUpdatedAt: 42, + observedAt: 100, + }); + }); + + it('rejects stale per-plane provenance when authoritative metadata is absent', () => { + const snapshot = describeContextGraphConvergence({ + readiness: { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: 42, + }, + includeSharedMemory: true, + hasConfirmedMeta: false, + observedAt: 100, + }); + + expect(snapshot).toMatchObject({ + state: 'pending', + verified: { + metadata: false, + durable: false, + sharedMemory: false, + }, + missing: ['metadata', 'durable', 'sharedMemory'], + }); + }); + + it('marks a VM-only request complete without requiring SWM', () => { + const snapshot = describeContextGraphConvergence({ + readiness: { + version: 1, + durableVerified: true, + sharedMemoryVerified: false, + updatedAt: 42, + }, + includeSharedMemory: false, + hasConfirmedMeta: true, + observedAt: 100, + }); + + expect(snapshot).toMatchObject({ + state: 'complete', + missing: [], + required: { sharedMemory: false }, + }); + }); +}); diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index af3085728..155ec4238 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -146,6 +146,8 @@ describe('context graph subscribe readiness requires authoritative metadata', () result?: CatchupJobResult; includeSharedMemory?: boolean; syncMode?: unknown; + coalescedUpgradeResult?: CatchupJobResult; + simulateAutomaticRecovery?: boolean; readiness?: { version: number; durableVerified: boolean; @@ -166,6 +168,7 @@ describe('context graph subscribe readiness requires authoritative metadata', () patches: Array>; readiness: Record | undefined; statusResponse: any; + coalescedResponse?: any; }> { const contextGraphId = `readiness-${Math.random().toString(36).slice(2, 8)}`; const state = new Map>(); @@ -184,12 +187,27 @@ describe('context graph subscribe readiness requires authoritative metadata', () let readiness = opts.readiness ? { ...opts.readiness, updatedAt: opts.readiness.updatedAt ?? Date.now() } : undefined; + let releaseFirstRun: (() => void) | undefined; + let noteFirstRunStarted: (() => void) | undefined; + const firstRunGate = opts.coalescedUpgradeResult + ? new Promise((resolve) => { releaseFirstRun = resolve; }) + : undefined; + const firstRunStarted = opts.coalescedUpgradeResult + ? new Promise((resolve) => { noteFirstRunStarted = resolve; }) + : undefined; daemonState.catchupRunner = { run: async (request) => { runCalls += 1; + const callNumber = runCalls; runRequests.push(request); - return opts.result ?? cleanEmptyResult(); + if (callNumber === 1 && firstRunGate) { + noteFirstRunStarted?.(); + await firstRunGate; + } + return callNumber === 2 && opts.coalescedUpgradeResult + ? opts.coalescedUpgradeResult + : opts.result ?? cleanEmptyResult(); }, close: async () => {}, }; @@ -291,11 +309,45 @@ describe('context graph subscribe readiness requires authoritative metadata', () const response = await httpResponse.json() as any; const jobId = response.catchup?.jobId as string | undefined; + let coalescedResponse: any; + if (firstRunStarted && releaseFirstRun) { + await firstRunStarted; + coalescedResponse = await fetch( + `http://127.0.0.1:${address.port}/api/context-graph/subscribe`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + contextGraphId, + includeSharedMemory: true, + ...(opts.syncMode !== undefined ? { syncMode: opts.syncMode } : {}), + }), + }, + ).then((result) => result.json()); + releaseFirstRun(); + } + for (let i = 0; jobId && i < 50; i++) { if (catchupTracker.jobs.get(jobId)?.finishedAt) break; await new Promise((resolve) => setTimeout(resolve, 5)); } + if (opts.simulateAutomaticRecovery) { + readiness = { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: Date.now(), + }; + state.set(contextGraphId, { + ...state.get(contextGraphId), + synced: true, + sharedMemorySynced: true, + metaSynced: true, + pendingMeta: false, + }); + } + const statusResponse = jobId ? await fetch( `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, @@ -313,6 +365,7 @@ describe('context graph subscribe readiness requires authoritative metadata', () patches, readiness, statusResponse, + coalescedResponse, }; } @@ -590,6 +643,17 @@ describe('context graph subscribe readiness requires authoritative metadata', () expect(result.statusResponse).toMatchObject({ jobId: result.response.catchup.jobId, status: 'done', + attemptStatus: 'done', + convergence: { + state: 'complete', + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + automaticRetryActive: true, + }, result: { dataSynced: 3, sharedMemorySynced: 4, @@ -772,15 +836,17 @@ describe('context graph subscribe readiness requires authoritative metadata', () }); expect(result.runCalls).toBe(1); - expect(result.job.status).toBe('done'); - expect(result.job.error).toBeUndefined(); + expect(result.job).toMatchObject({ + status: 'unreachable', + error: expect.stringContaining('durable VM'), + }); expect(result.job.result).toMatchObject({ dataSynced: 0, sharedMemorySynced: 4, }); expect(result.state).toMatchObject({ subscribed: true, - synced: true, + synced: false, sharedMemorySynced: true, metaSynced: true, pendingMeta: false, @@ -791,6 +857,81 @@ describe('context graph subscribe readiness requires authoritative metadata', () }); }); + it('coalesces a running VM-only selection and serially upgrades it to VM plus SWM', async () => { + const result = await subscribe({ + hasConfirmedMeta: true, + includeSharedMemory: false, + result: privateDataOnlyResult(), + coalescedUpgradeResult: publicDurableAndSharedMemoryResult(), + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + }); + + expect(result.coalescedResponse).toMatchObject({ + subscribed: result.response.subscribed, + catchup: { + status: 'running', + includeWorkspace: true, + jobId: result.response.catchup.jobId, + }, + }); + expect(result.runRequests).toEqual([ + { + contextGraphId: result.response.subscribed, + includeSharedMemory: false, + }, + { + contextGraphId: result.response.subscribed, + includeSharedMemory: true, + }, + ]); + expect(result.job).toMatchObject({ + jobId: result.response.catchup.jobId, + includeWorkspace: true, + status: 'done', + }); + expect(result.statusResponse.convergence).toMatchObject({ + state: 'complete', + missing: [], + }); + }); + + it('reports live completion when automatic retry recovers a failed foreground attempt', async () => { + const result = await subscribe({ + hasConfirmedMeta: true, + isPrivate: true, + result: privateSharedMemoryOnlyResult(), + simulateAutomaticRecovery: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + }); + + expect(result.job.status).toBe('unreachable'); + expect(result.statusResponse).toMatchObject({ + status: 'done', + attemptStatus: 'unreachable', + attemptError: expect.stringContaining('durable VM'), + completedAfterAttempt: true, + convergence: { + state: 'complete', + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + }, + }); + }); + it('does not promote positive durable inserts when the plane also timed out', async () => { const partial = privateDataOnlyResult(); if (!partial.diagnostics?.durable) throw new Error('durable diagnostics missing'); From 1ed176927eef8691d059bd39c7d23e8a359bbff0 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 03:21:03 +0200 Subject: [PATCH 12/23] fix(sync): preserve coalesced catchup job scope --- packages/cli/src/context-graph-readiness.ts | 77 +++++++--- .../cli/src/daemon/routes/context-graph.ts | 141 +++++++++++++----- packages/cli/src/daemon/types.ts | 13 ++ .../context-graph-subscribe-readiness.test.ts | 88 ++++++++++- 4 files changed, 254 insertions(+), 65 deletions(-) diff --git a/packages/cli/src/context-graph-readiness.ts b/packages/cli/src/context-graph-readiness.ts index 6794681e9..85543a607 100644 --- a/packages/cli/src/context-graph-readiness.ts +++ b/packages/cli/src/context-graph-readiness.ts @@ -61,17 +61,22 @@ export interface ContextGraphConvergenceSnapshot { observedAt: number; } +export interface ContextGraphReadinessPlanes + extends Omit { + currentReadinessProvenance: boolean; +} + /** - * Describe the live, per-plane convergence of one selected context graph. - * A VM/SWM proof is only effective while authoritative CG metadata is local; - * stale provenance must not make a graph look complete after metadata loss. + * Canonical interpretation of persisted readiness as independently verified + * metadata, durable VM, and optional SWM planes. Every caller that needs to + * decide whether a selected graph is ready must go through this helper so + * metadata loss and readiness-version changes cannot drift between paths. */ -export function describeContextGraphConvergence(input: { +export function describeReadinessPlanes(input: { readiness: ContextGraphReadinessProvenance; includeSharedMemory: boolean; hasConfirmedMeta: boolean; - observedAt?: number; -}): ContextGraphConvergenceSnapshot { +}): ContextGraphReadinessPlanes { const currentReadinessProvenance = input.readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; const metadataVerified = input.hasConfirmedMeta; @@ -90,6 +95,7 @@ export function describeContextGraphConvergence(input: { (input.includeSharedMemory && sharedMemoryVerified); return { + currentReadinessProvenance, state: missing.length === 0 ? 'complete' : anyVerified ? 'partial' : 'pending', required: { metadata: true, @@ -105,6 +111,27 @@ export function describeContextGraphConvergence(input: { ...(currentReadinessProvenance ? { readinessUpdatedAt: input.readiness.updatedAt } : {}), + }; +} + +/** + * Describe the live, per-plane convergence of one selected context graph. + * A VM/SWM proof is only effective while authoritative CG metadata is local; + * stale provenance must not make a graph look complete after metadata loss. + */ +export function describeContextGraphConvergence(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; + observedAt?: number; +}): ContextGraphConvergenceSnapshot { + const { + currentReadinessProvenance: _currentReadinessProvenance, + ...planes + } = describeReadinessPlanes(input); + + return { + ...planes, observedAt: input.observedAt ?? Date.now(), }; } @@ -140,15 +167,9 @@ export function classifyExistingContextGraphReadiness(input: { statePatch?: ContextGraphSubscriptionStatePatch; readinessPatch?: ContextGraphReadinessPatch; } { - const currentReadinessProvenance = - input.readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; - const requestedPlanesVerified = - currentReadinessProvenance && - input.readiness.durableVerified && - (!input.includeSharedMemory || input.readiness.sharedMemoryVerified); + const planes = describeReadinessPlanes(input); const alreadyReady = - input.hasConfirmedMeta && - requestedPlanesVerified && + planes.state === 'complete' && input.subscription.synced === true && (!input.includeSharedMemory || input.subscription.sharedMemorySynced === true); @@ -174,10 +195,8 @@ export function classifyExistingContextGraphReadiness(input: { }; } - const durableVerified = - currentReadinessProvenance && input.readiness.durableVerified; - const sharedMemoryVerified = - currentReadinessProvenance && input.readiness.sharedMemoryVerified; + const durableVerified = planes.verified.durable; + const sharedMemoryVerified = planes.verified.sharedMemory; const statePatch = input.subscription.synced !== durableVerified || input.subscription.sharedMemorySynced !== sharedMemoryVerified @@ -192,7 +211,7 @@ export function classifyExistingContextGraphReadiness(input: { return { alreadyReady: false, statePatch, - readinessPatch: currentReadinessProvenance + readinessPatch: planes.currentReadinessProvenance ? undefined : { durableVerified: false, @@ -407,17 +426,27 @@ export function classifyContextGraphCatchupReadiness(input: { currentReadinessProvenance && input.readinessBeforeCatchup.durableVerified; const sharedMemoryVerifiedBefore = currentReadinessProvenance && input.readinessBeforeCatchup.sharedMemoryVerified; - const durableVerified = durableVerifiedBefore || durableReadyThisRun; - const sharedMemoryVerified = sharedMemoryVerifiedBefore || sharedMemoryReadyThisRun; + const planes = describeReadinessPlanes({ + readiness: { + version: CONTEXT_GRAPH_READINESS_VERSION, + durableVerified: durableVerifiedBefore || durableReadyThisRun, + sharedMemoryVerified: + sharedMemoryVerifiedBefore || sharedMemoryReadyThisRun, + updatedAt: input.readinessBeforeCatchup.updatedAt, + }, + includeSharedMemory: input.includeSharedMemory, + hasConfirmedMeta: input.hasConfirmedMeta, + }); + const durableVerified = planes.verified.durable; + const sharedMemoryVerified = planes.verified.sharedMemory; // What this run is allowed to FREEZE, as opposed to what it reports. These // diverge only for a unanimous-empty verdict, which stays re-derived per run // so that a wrong empty verdict cannot become permanent. const durableVerifiedPersisted = durableVerifiedBefore || durableThisRun.persistable; const sharedMemoryVerifiedPersisted = sharedMemoryVerifiedBefore || sharedMemoryThisRun.persistable; - const missingDurable = !durableVerified; - const missingRequestedSharedMemory = - input.includeSharedMemory && !sharedMemoryVerified; + const missingDurable = planes.missing.includes('durable'); + const missingRequestedSharedMemory = planes.missing.includes('sharedMemory'); const madeIncompleteProgress = (durableDataProgress && !durableReadyThisRun) || (sharedMemoryProgress && !sharedMemoryReadyThisRun); diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index f05dc6a95..b43e852ec 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -167,6 +167,7 @@ import { import { type CatchupJobState, type CatchupJob, + type CatchupCoordinator, type CatchupTracker, toCatchupStatusResponse, } from '../types.js'; @@ -1741,25 +1742,53 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise(); + const existingCoordinator = inFlightByContextGraph.get(contextGraphId); const existingJobId = catchupTracker.latestByContextGraph.get(contextGraphId); const existingJob = existingJobId ? catchupTracker.jobs.get(existingJobId) : undefined; let readinessBeforeCatchup = readContextGraphReadiness(dashDb, contextGraphId); if (existingSub?.subscribed) { - if (existingJob && (existingJob.status === "queued" || existingJob.status === "running")) { - // Coalesce repeated selection into the one in-flight job. Required - // planes are monotonic: a later VM+SWM request upgrades a VM-only - // attempt, while a narrower request can never remove SWM from work - // that is already queued or running. - existingJob.includeWorkspace = existingJob.includeWorkspace || - shouldSyncSharedMemory; + const baseJob = existingCoordinator + ? catchupTracker.jobs.get(existingCoordinator.baseJobId) + : undefined; + const upgradeJob = existingCoordinator?.upgradeJobId + ? catchupTracker.jobs.get(existingCoordinator.upgradeJobId) + : undefined; + const coordinatorActive = existingCoordinator && + [baseJob, upgradeJob].some((candidate) => + candidate?.status === 'queued' || candidate?.status === 'running'); + if (existingCoordinator && baseJob && coordinatorActive) { + // Coalesce onto one serialized worker without changing the contract of + // an already-issued jobId. A later VM+SWM request gets a separate + // public job record while the coordinator records the monotonic work + // upgrade for the running worker. + let responseJob = baseJob; + if (shouldSyncSharedMemory && !baseJob.includeWorkspace) { + existingCoordinator.requestedIncludeSharedMemory = true; + responseJob = upgradeJob ?? (() => { + const upgradeJobId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; + const createdUpgradeJob: CatchupJob = { + jobId: upgradeJobId, + contextGraphId, + includeWorkspace: true, + status: 'queued', + queuedAt: Date.now(), + }; + catchupTracker.jobs.set(upgradeJobId, createdUpgradeJob); + catchupTracker.latestByContextGraph.set(contextGraphId, upgradeJobId); + existingCoordinator.upgradeJobId = upgradeJobId; + return createdUpgradeJob; + })(); + } return jsonResponse(res, 200, { subscribed: contextGraphId, syncMode: effectiveSyncMode, catchup: { - status: existingJob.status, - includeWorkspace: existingJob.includeWorkspace, - jobId: existingJob.jobId, + status: responseJob.status, + includeWorkspace: responseJob.includeWorkspace, + jobId: responseJob.jobId, }, }); } @@ -1777,7 +1806,10 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise 100) { let oldestId: string | undefined; @@ -1859,28 +1897,44 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise { - job.status = "running"; - job.startedAt = Date.now(); - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} started`); + let attemptJob = job; + const settleQueuedUpgradeFrom = (source: CatchupJob) => { + const queuedUpgrade = coordinator.upgradeJobId + ? catchupTracker.jobs.get(coordinator.upgradeJobId) + : undefined; + if (!queuedUpgrade || queuedUpgrade.status !== 'queued') return; + queuedUpgrade.status = source.status; + queuedUpgrade.error = source.error; + queuedUpgrade.result = source.result; + queuedUpgrade.startedAt = source.startedAt; + queuedUpgrade.finishedAt = Date.now(); + }; try { let attemptReadinessBeforeCatchup = readinessBeforeCatchup; while (true) { - const attemptIncludeSharedMemory = job.includeWorkspace; + attemptJob.status = 'running'; + attemptJob.startedAt ??= Date.now(); + const attemptIncludeSharedMemory = attemptJob.includeWorkspace; + if (DEBUG_SYNC_TRACE) { + console.log( + `[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} started`, + ); + } const result = await daemonState.catchupRunner!.run({ contextGraphId: contextGraphId, includeSharedMemory: attemptIncludeSharedMemory, }); - job.result = result; - job.error = undefined; + attemptJob.result = result; + attemptJob.error = undefined; // Local scheduler pressure cut the round short. An incomplete round has // no readiness to inspect and must never finalize the subscription, so // short-circuit the whole classification path and report a distinct // retryable status. A remote denial still wins: waiting for local // capacity will never clear it. if (result.deferredBackpressure > 0 && !result.denied) { - job.status = "deferred"; - job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} deferred by local scheduler: ${result.deferredBackpressure}`); + attemptJob.status = "deferred"; + attemptJob.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; + if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} deferred by local scheduler: ${result.deferredBackpressure}`); } else { const inspectReadiness = catchupResultHasCleanResponse(result); const hasConfirmedMeta = inspectReadiness @@ -1897,8 +1951,8 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise 0) { - job.status = "deferred"; - job.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; + if (attemptJob.status === "done" && result.deferredBackpressure > 0) { + attemptJob.status = "deferred"; + attemptJob.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; } } + attemptJob.finishedAt = Date.now(); if (DEBUG_SYNC_TRACE) { - if (job.status === 'denied') { - console.log(`[catchup] job=${jobId} contextGraph=${contextGraphId} denied by remote peer(s): ${result.deniedPeers}`); + if (attemptJob.status === 'denied') { + console.log(`[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} denied by remote peer(s): ${result.deniedPeers}`); } console.log( - `[catchup] job=${jobId} contextGraph=${contextGraphId} status=${job.status} ` + + `[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} status=${attemptJob.status} ` + `peers=${result.peersTried}/${result.syncCapablePeers} ` + `connected=${result.totalPeers ?? result.connectedPeers} ` + `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, @@ -1940,29 +1995,39 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise; latestByContextGraph: Map; + inFlightByContextGraph?: Map; } export interface CatchupConvergenceStatus extends ContextGraphConvergenceSnapshot { diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index 155ec4238..c28726590 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -169,6 +169,8 @@ describe('context graph subscribe readiness requires authoritative metadata', () readiness: Record | undefined; statusResponse: any; coalescedResponse?: any; + coalescedJob?: any; + coalescedStatusResponse?: any; }> { const contextGraphId = `readiness-${Math.random().toString(36).slice(2, 8)}`; const state = new Map>(); @@ -327,8 +329,13 @@ describe('context graph subscribe readiness requires authoritative metadata', () releaseFirstRun(); } + const coalescedJobId = coalescedResponse?.catchup?.jobId as string | undefined; + for (let i = 0; jobId && i < 50; i++) { - if (catchupTracker.jobs.get(jobId)?.finishedAt) break; + const originalFinished = catchupTracker.jobs.get(jobId)?.finishedAt; + const coalescedFinished = !coalescedJobId || + catchupTracker.jobs.get(coalescedJobId)?.finishedAt; + if (originalFinished && coalescedFinished) break; await new Promise((resolve) => setTimeout(resolve, 5)); } @@ -353,6 +360,11 @@ describe('context graph subscribe readiness requires authoritative metadata', () `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, ).then((result) => result.json()) : null; + const coalescedStatusResponse = coalescedJobId + ? await fetch( + `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(coalescedJobId)}`, + ).then((result) => result.json()) + : null; return { response, @@ -366,6 +378,10 @@ describe('context graph subscribe readiness requires authoritative metadata', () readiness, statusResponse, coalescedResponse, + coalescedJob: coalescedJobId + ? catchupTracker.jobs.get(coalescedJobId) + : undefined, + coalescedStatusResponse, }; } @@ -874,11 +890,13 @@ describe('context graph subscribe readiness requires authoritative metadata', () expect(result.coalescedResponse).toMatchObject({ subscribed: result.response.subscribed, catchup: { - status: 'running', + status: 'queued', includeWorkspace: true, - jobId: result.response.catchup.jobId, }, }); + expect(result.coalescedResponse.catchup.jobId).not.toBe( + result.response.catchup.jobId, + ); expect(result.runRequests).toEqual([ { contextGraphId: result.response.subscribed, @@ -891,13 +909,77 @@ describe('context graph subscribe readiness requires authoritative metadata', () ]); expect(result.job).toMatchObject({ jobId: result.response.catchup.jobId, + includeWorkspace: false, + status: 'done', + }); + expect(result.coalescedJob).toMatchObject({ + jobId: result.coalescedResponse.catchup.jobId, includeWorkspace: true, status: 'done', }); expect(result.statusResponse.convergence).toMatchObject({ state: 'complete', + required: { sharedMemory: false }, missing: [], }); + expect(result.coalescedStatusResponse.convergence).toMatchObject({ + state: 'complete', + required: { sharedMemory: true }, + missing: [], + }); + }); + + it('keeps VM-only success stable when a coalesced SWM upgrade fails', async () => { + const result = await subscribe({ + hasConfirmedMeta: true, + includeSharedMemory: false, + result: privateDataOnlyResult(), + coalescedUpgradeResult: privateDataOnlyResult(), + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + }); + + expect(result.runRequests).toEqual([ + { + contextGraphId: result.response.subscribed, + includeSharedMemory: false, + }, + { + contextGraphId: result.response.subscribed, + includeSharedMemory: true, + }, + ]); + expect(result.job).toMatchObject({ + includeWorkspace: false, + status: 'done', + }); + expect(result.statusResponse).toMatchObject({ + status: 'done', + includeSharedMemory: false, + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + expect(result.coalescedJob).toMatchObject({ + includeWorkspace: true, + status: 'unreachable', + error: expect.stringContaining('requested data plane'), + }); + expect(result.coalescedStatusResponse).toMatchObject({ + status: 'unreachable', + includeSharedMemory: true, + convergence: { + state: 'partial', + required: { sharedMemory: true }, + missing: ['sharedMemory'], + }, + }); }); it('reports live completion when automatic retry recovers a failed foreground attempt', async () => { From 20c554bdcddd3ac54cf37ad04eccddb192c70948 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 03:49:24 +0200 Subject: [PATCH 13/23] fix(sync): harden catch-up convergence --- packages/cli/src/api-client.ts | 87 +-- packages/cli/src/catchup-result-wire.ts | 76 +++ packages/cli/src/catchup-runner.ts | 96 +--- packages/cli/src/catchup-status-wire.ts | 43 ++ packages/cli/src/cli-helpers.ts | 6 +- .../cli/src/context-graph-readiness-wire.ts | 20 + packages/cli/src/context-graph-readiness.ts | 171 ++++-- .../cli/src/daemon/catchup-status-response.ts | 50 ++ .../context-graph-catchup-coordinator.ts | 325 +++++++++++ packages/cli/src/daemon/handle-request.ts | 1 - packages/cli/src/daemon/lifecycle.ts | 2 +- packages/cli/src/daemon/routes/agent-chat.ts | 1 - .../cli/src/daemon/routes/context-graph.ts | 246 ++------ packages/cli/src/daemon/routes/epcis.ts | 1 - .../cli/src/daemon/routes/local-agents.ts | 1 - packages/cli/src/daemon/routes/memory.ts | 1 - packages/cli/src/daemon/routes/openclaw.ts | 1 - packages/cli/src/daemon/routes/publisher.ts | 1 - packages/cli/src/daemon/routes/query.ts | 9 +- packages/cli/src/daemon/routes/status.ts | 1 - packages/cli/src/daemon/types.ts | 54 +- packages/cli/test/catchup-status-cli.test.ts | 64 +++ .../catchup-status-convergence-route.test.ts | 143 +++++ .../cli/test/catchup-status-response.test.ts | 43 +- ...ext-graph-catchup-coalescing-route.test.ts | 218 +++++++ .../context-graph-catchup-coordinator.test.ts | 230 ++++++++ .../context-graph-catchup-readiness.test.ts | 31 + .../context-graph-subscribe-readiness.test.ts | 539 +----------------- .../test/daemon-http-behavior-extra.test.ts | 13 +- .../helpers/context-graph-catchup-fixtures.ts | 119 ++++ .../context-graph-subscribe-route-harness.ts | 337 +++++++++++ packages/cli/vitest.unit.config.ts | 5 + 32 files changed, 1886 insertions(+), 1049 deletions(-) create mode 100644 packages/cli/src/catchup-result-wire.ts create mode 100644 packages/cli/src/catchup-status-wire.ts create mode 100644 packages/cli/src/context-graph-readiness-wire.ts create mode 100644 packages/cli/src/daemon/catchup-status-response.ts create mode 100644 packages/cli/src/daemon/context-graph-catchup-coordinator.ts create mode 100644 packages/cli/test/catchup-status-cli.test.ts create mode 100644 packages/cli/test/catchup-status-convergence-route.test.ts create mode 100644 packages/cli/test/context-graph-catchup-coalescing-route.test.ts create mode 100644 packages/cli/test/context-graph-catchup-coordinator.test.ts create mode 100644 packages/cli/test/helpers/context-graph-catchup-fixtures.ts create mode 100644 packages/cli/test/helpers/context-graph-subscribe-route-harness.ts diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index 21d6e034a..2c585c3f4 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -14,6 +14,7 @@ import { } from './finalized-publish-options.js'; import type { RegisterPcaAgentResult } from './pca-confirmation-wire.js'; import { parseRegisterPcaAgentResult } from './pca-confirmation-wire.js'; +import type { CatchupStatusResponse } from './catchup-status-wire.js'; export type { KnowledgeAssetFinalizedPublishOptions } from './finalized-publish-options.js'; @@ -1643,91 +1644,7 @@ export class ApiClient { }); } - async catchupStatus(contextGraphId: string): Promise<{ - jobId: string; - contextGraphId: string; - includeWorkspace: boolean; - status: 'queued' | 'running' | 'done' | 'denied' | 'deferred' | 'failed' | 'unreachable'; - attemptStatus: 'queued' | 'running' | 'done' | 'denied' | 'deferred' | 'failed' | 'unreachable'; - queuedAt: number; - startedAt?: number; - finishedAt?: number; - result?: { - connectedPeers: number; - totalPeers?: number; - selectedPeers?: number; - syncCapablePeers: number; - peersTried: number; - peersResponded: number; - peersSucceeded: number; - /** Sync-capable peers skipped because an earlier wave already proved every requested plane. */ - peersNotAttempted?: number; - deferredBackpressure: number; - dataSynced: number; - sharedMemorySynced: number; - denied: boolean; - deniedPeers: number; - diagnostics?: { - noProtocolPeers: number; - durable: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - metaOnlyResponses: number; - verifiedPrivateOnlyResponses?: number; - dataRejectedMissingMeta: number; - rejectedKcs: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - }; - sharedMemory: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - droppedDataTriples: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - }; - }; - }; - error?: string; - attemptError?: string; - convergence?: { - state: 'pending' | 'partial' | 'complete'; - required: { - metadata: true; - durable: true; - sharedMemory: boolean; - }; - verified: { - metadata: boolean; - durable: boolean; - sharedMemory: boolean; - }; - missing: Array<'metadata' | 'durable' | 'sharedMemory'>; - readinessUpdatedAt?: number; - observedAt: number; - syncMode: 'on-demand' | 'always-on'; - automaticRetryActive: boolean; - }; - completedAfterAttempt?: true; - }> { + async catchupStatus(contextGraphId: string): Promise { return this.get(`/api/sync/catchup-status?contextGraphId=${encodeURIComponent(contextGraphId)}`); } diff --git a/packages/cli/src/catchup-result-wire.ts b/packages/cli/src/catchup-result-wire.ts new file mode 100644 index 000000000..460252fc0 --- /dev/null +++ b/packages/cli/src/catchup-result-wire.ts @@ -0,0 +1,76 @@ +/** Per-plane clean-completion evidence safe to expose on the status wire. */ +export interface CatchupPlaneCompletionEvidence { + verifiedDataPeers: number; + verifiedPrivateOnlyPeers?: number; + emptyPeers: number; + authorityEmptyPeers?: number; + incompleteResponders?: number; +} + +/** Stable per-attempt result exposed by the catch-up status API. */ +export interface CatchupJobResult { + connectedPeers: number; + totalPeers?: number; + selectedPeers?: number; + syncCapablePeers: number; + peersTried: number; + /** Peers that reached a responder without collapsing into transport failure. */ + peersResponded: number; + /** Peers whose requested sync round completed cleanly. */ + peersSucceeded: number; + /** Sync-capable peers skipped after an earlier wave proved every requested plane. */ + peersNotAttempted?: number; + /** Context Graph phases deferred by this node's local sync scheduler. */ + deferredBackpressure: number; + dataSynced: number; + sharedMemorySynced: number; + denied: boolean; + deniedPeers: number; + cleanPlaneCompletions?: { + durable: CatchupPlaneCompletionEvidence & { verifiedPrivateOnlyPeers: number }; + sharedMemory: CatchupPlaneCompletionEvidence; + }; + diagnostics?: { + noProtocolPeers: number; + durable: { + fetchedMetaTriples: number; + fetchedDataTriples: number; + insertedMetaTriples: number; + insertedDataTriples: number; + bytesReceived: number; + resumedPhases: number; + timedOutPhases: number; + completedPhases: number; + checkpointAdvances: number; + emptyResponses: number; + metaOnlyResponses: number; + /** Cryptographically verified V2 responses whose public graph is intentionally empty. */ + verifiedPrivateOnlyResponses: number; + dataRejectedMissingMeta: number; + rejectedKcs: number; + failedPeers: number; + failedPhases: number; + deferredBackpressure: number; + deniedPhases?: number; + authorityUnanswered?: boolean; + }; + sharedMemory: { + fetchedMetaTriples: number; + fetchedDataTriples: number; + insertedMetaTriples: number; + insertedDataTriples: number; + bytesReceived: number; + resumedPhases: number; + timedOutPhases: number; + completedPhases: number; + checkpointAdvances: number; + emptyResponses: number; + droppedDataTriples: number; + failedPeers: number; + failedPhases: number; + deferredBackpressure: number; + deniedPhases?: number; + authorityUnanswered?: boolean; + }; + }; +} diff --git a/packages/cli/src/catchup-runner.ts b/packages/cli/src/catchup-runner.ts index 8c06daf29..cba5164fd 100644 --- a/packages/cli/src/catchup-runner.ts +++ b/packages/cli/src/catchup-runner.ts @@ -13,6 +13,9 @@ import { type SyncPeerResolution, } from '@origintrail-official/dkg-agent'; import { PROTOCOL_SYNC } from '@origintrail-official/dkg-core'; +import type { CatchupJobResult } from './catchup-result-wire.js'; + +export type { CatchupJobResult } from './catchup-result-wire.js'; const SYNC_PROTOCOL_CHECK_ATTEMPTS = 3; const SYNC_PROTOCOL_CHECK_DELAY_MS = 500; @@ -20,99 +23,6 @@ const DURABLE_CATCHUP_PHASE_HEADROOM_MS = 1_000; const MIN_DURABLE_CATCHUP_PHASE_BUDGET_MS = 1_000; const DURABLE_CATCHUP_SETTLEMENT_GRACE_MS = 30_000; -export interface CatchupJobResult { - connectedPeers: number; - totalPeers?: number; - selectedPeers?: number; - syncCapablePeers: number; - peersTried: number; - /** - * Subset of `peersTried` whose per-peer sync round reached a responder - * and did not collapse into a transport failure. A responder can still - * time out part-way through, deny access, or serve metadata-only rows; this - * counter exists so daemon status mapping can distinguish "curator offline" - * from "reachable peer answered but did not complete cleanly". - */ - peersResponded: number; - /** - * Subset of `peersTried` whose per-peer sync round finished without a - * transport failure, timeout, or explicit ACL denial, and with either real - * progress or a clean non-metadata-only empty completion. - */ - peersSucceeded: number; - /** - * Sync-capable peers this run deliberately never contacted because an earlier - * wave already proved every requested plane. These are neither failures nor - * successes; they exist so status mapping and operators can tell an - * early-stopped run from a run where peers were unreachable. - */ - peersNotAttempted?: number; - /** Context Graph phases deferred by this node's local sync scheduler. */ - deferredBackpressure: number; - dataSynced: number; - sharedMemorySynced: number; - denied: boolean; - deniedPeers: number; - /** - * Per-plane evidence produced before peer results are aggregated. Aggregate - * diagnostics intentionally retain every timeout/denial for observability, - * but readiness must not let one bad peer mask another peer that completed - * the same plane cleanly and stored verified data. - */ - cleanPlaneCompletions?: { - /** Always carries `verifiedPrivateOnlyPeers`; only the durable plane can produce it. */ - durable: CatchupPlaneCompletionEvidence & { verifiedPrivateOnlyPeers: number }; - sharedMemory: CatchupPlaneCompletionEvidence; - }; - diagnostics?: { - noProtocolPeers: number; - durable: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - metaOnlyResponses: number; - /** Cryptographically verified V2 responses whose public graph is intentionally empty. */ - verifiedPrivateOnlyResponses: number; - dataRejectedMissingMeta: number; - rejectedKcs: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - deniedPhases?: number; - /** A resolvable curator never cleanly answered this plane; see - * `catchupPlaneProvenByUnanimousEmpty`. */ - authorityUnanswered?: boolean; - }; - sharedMemory: { - fetchedMetaTriples: number; - fetchedDataTriples: number; - insertedMetaTriples: number; - insertedDataTriples: number; - bytesReceived: number; - resumedPhases: number; - timedOutPhases: number; - completedPhases: number; - checkpointAdvances: number; - emptyResponses: number; - droppedDataTriples: number; - failedPeers: number; - failedPhases: number; - deferredBackpressure: number; - deniedPhases?: number; - /** A resolvable curator never cleanly answered this plane; see - * `catchupPlaneProvenByUnanimousEmpty`. */ - authorityUnanswered?: boolean; - }; - }; -} - export interface CatchupRunRequest { contextGraphId: string; includeSharedMemory: boolean; diff --git a/packages/cli/src/catchup-status-wire.ts b/packages/cli/src/catchup-status-wire.ts new file mode 100644 index 000000000..5570f8fe0 --- /dev/null +++ b/packages/cli/src/catchup-status-wire.ts @@ -0,0 +1,43 @@ +import type { CatchupJobResult } from './catchup-result-wire.js'; +import type { ContextGraphConvergenceSnapshot } from './context-graph-readiness-wire.js'; + +export type CatchupJobState = + | 'queued' + | 'running' + | 'done' + | 'failed' + | 'denied' + /** Local scheduler capacity was unavailable; retry is safe. */ + | 'deferred' + /** + * Peers were reachable, but none delivered every required plane during this + * attempt. This remains distinct from authorization denial and worker error. + */ + | 'unreachable'; + +export interface CatchupConvergenceStatus extends ContextGraphConvergenceSnapshot { + syncMode: 'on-demand' | 'always-on'; + automaticRetryActive: boolean; +} + +/** Shared daemon/client contract for the catch-up status endpoint. */ +export interface CatchupStatusResponse { + jobId: string; + contextGraphId: string; + includeWorkspace: boolean; + includeSharedMemory: boolean; + /** Canonical actionable status, including newer live convergence evidence. */ + status: CatchupJobState; + queuedAt: number; + startedAt?: number; + finishedAt?: number; + result?: CatchupJobResult; + error?: string; + /** Historical runner outcome, emitted only when it differs from status. */ + attempt?: { + status: CatchupJobState; + error?: string; + }; + convergence?: CatchupConvergenceStatus; + completedAfterAttempt?: true; +} diff --git a/packages/cli/src/cli-helpers.ts b/packages/cli/src/cli-helpers.ts index 8f307241c..f09b86ffa 100644 --- a/packages/cli/src/cli-helpers.ts +++ b/packages/cli/src/cli-helpers.ts @@ -202,9 +202,9 @@ function printCatchupStatus(status: Awaited; -export interface ContextGraphReadinessPlanes - extends Omit { - currentReadinessProvenance: boolean; +function hasCurrentReadinessProvenance( + readiness: ContextGraphReadinessProvenance, +): boolean { + return readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; } -/** - * Canonical interpretation of persisted readiness as independently verified - * metadata, durable VM, and optional SWM planes. Every caller that needs to - * decide whether a selected graph is ready must go through this helper so - * metadata loss and readiness-version changes cannot drift between paths. - */ -export function describeReadinessPlanes(input: { - readiness: ContextGraphReadinessProvenance; - includeSharedMemory: boolean; +function describeResolvedReadinessPlanes(input: { hasConfirmedMeta: boolean; + includeSharedMemory: boolean; + durableEvidence: boolean; + sharedMemoryEvidence: boolean; + readinessUpdatedAt?: number; }): ContextGraphReadinessPlanes { - const currentReadinessProvenance = - input.readiness.version >= CONTEXT_GRAPH_READINESS_VERSION; const metadataVerified = input.hasConfirmedMeta; - const durableVerified = metadataVerified && - currentReadinessProvenance && - input.readiness.durableVerified; - const sharedMemoryVerified = metadataVerified && - currentReadinessProvenance && - input.readiness.sharedMemoryVerified; + const durableVerified = metadataVerified && input.durableEvidence; + const sharedMemoryVerified = metadataVerified && input.sharedMemoryEvidence; const missing: ContextGraphConvergencePlane[] = []; if (!metadataVerified) missing.push('metadata'); if (!durableVerified) missing.push('durable'); @@ -95,7 +80,6 @@ export function describeReadinessPlanes(input: { (input.includeSharedMemory && sharedMemoryVerified); return { - currentReadinessProvenance, state: missing.length === 0 ? 'complete' : anyVerified ? 'partial' : 'pending', required: { metadata: true, @@ -108,10 +92,66 @@ export function describeReadinessPlanes(input: { sharedMemory: sharedMemoryVerified, }, missing, + ...(input.readinessUpdatedAt !== undefined + ? { readinessUpdatedAt: input.readinessUpdatedAt } + : {}), + }; +} + +/** + * Canonical interpretation of persisted readiness as independently verified + * metadata, durable VM, and optional SWM planes. Every caller that needs to + * decide whether a selected graph is ready must go through this helper so + * metadata loss and readiness-version changes cannot drift between paths. + */ +export function describeReadinessPlanes(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ContextGraphReadinessPlanes { + const currentReadinessProvenance = hasCurrentReadinessProvenance(input.readiness); + return describeResolvedReadinessPlanes({ + hasConfirmedMeta: input.hasConfirmedMeta, + includeSharedMemory: input.includeSharedMemory, + durableEvidence: + currentReadinessProvenance && input.readiness.durableVerified, + sharedMemoryEvidence: + currentReadinessProvenance && input.readiness.sharedMemoryVerified, ...(currentReadinessProvenance ? { readinessUpdatedAt: input.readiness.updatedAt } : {}), - }; + }); +} + +/** + * Merge current-run completion evidence with persisted provenance without + * pretending the merged value was itself read from storage. + */ +export function combineCatchupPlaneEvidence(input: { + readinessBeforeCatchup: ContextGraphReadinessProvenance; + durableReadyThisRun: boolean; + sharedMemoryReadyThisRun: boolean; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ContextGraphReadinessPlanes { + const currentReadinessProvenance = hasCurrentReadinessProvenance( + input.readinessBeforeCatchup, + ); + return describeResolvedReadinessPlanes({ + hasConfirmedMeta: input.hasConfirmedMeta, + includeSharedMemory: input.includeSharedMemory, + durableEvidence: + (currentReadinessProvenance && + input.readinessBeforeCatchup.durableVerified) || + input.durableReadyThisRun, + sharedMemoryEvidence: + (currentReadinessProvenance && + input.readinessBeforeCatchup.sharedMemoryVerified) || + input.sharedMemoryReadyThisRun, + ...(currentReadinessProvenance + ? { readinessUpdatedAt: input.readinessBeforeCatchup.updatedAt } + : {}), + }); } /** @@ -125,13 +165,8 @@ export function describeContextGraphConvergence(input: { hasConfirmedMeta: boolean; observedAt?: number; }): ContextGraphConvergenceSnapshot { - const { - currentReadinessProvenance: _currentReadinessProvenance, - ...planes - } = describeReadinessPlanes(input); - return { - ...planes, + ...describeReadinessPlanes(input), observedAt: input.observedAt ?? Date.now(), }; } @@ -167,6 +202,9 @@ export function classifyExistingContextGraphReadiness(input: { statePatch?: ContextGraphSubscriptionStatePatch; readinessPatch?: ContextGraphReadinessPatch; } { + const currentReadinessProvenance = hasCurrentReadinessProvenance( + input.readiness, + ); const planes = describeReadinessPlanes(input); const alreadyReady = planes.state === 'complete' && @@ -211,7 +249,7 @@ export function classifyExistingContextGraphReadiness(input: { return { alreadyReady: false, statePatch, - readinessPatch: planes.currentReadinessProvenance + readinessPatch: currentReadinessProvenance ? undefined : { durableVerified: false, @@ -426,14 +464,10 @@ export function classifyContextGraphCatchupReadiness(input: { currentReadinessProvenance && input.readinessBeforeCatchup.durableVerified; const sharedMemoryVerifiedBefore = currentReadinessProvenance && input.readinessBeforeCatchup.sharedMemoryVerified; - const planes = describeReadinessPlanes({ - readiness: { - version: CONTEXT_GRAPH_READINESS_VERSION, - durableVerified: durableVerifiedBefore || durableReadyThisRun, - sharedMemoryVerified: - sharedMemoryVerifiedBefore || sharedMemoryReadyThisRun, - updatedAt: input.readinessBeforeCatchup.updatedAt, - }, + const planes = combineCatchupPlaneEvidence({ + readinessBeforeCatchup: input.readinessBeforeCatchup, + durableReadyThisRun, + sharedMemoryReadyThisRun, includeSharedMemory: input.includeSharedMemory, hasConfirmedMeta: input.hasConfirmedMeta, }); @@ -588,6 +622,24 @@ async function withContextGraphReadinessMutationLock( } } +/** + * Require authoritative metadata for a readiness decision. Only a locally + * curated graph may trust its own pre-registration definition; remote graphs + * must reject the legacy unregistered placeholder shape. + */ +export async function hasAuthoritativeContextGraphMetadata(input: { + agent: DKGAgent; + contextGraphId: string; +}): Promise { + const locallyCurated = typeof input.agent.isCuratorOf === 'function' + ? await input.agent.isCuratorOf(input.contextGraphId).catch(() => false) + : false; + return input.agent.hasConfirmedMetaState( + input.contextGraphId, + { rejectUnregisteredPlaceholder: !locallyCurated }, + ).catch(() => false); +} + /** * Revalidate live metadata and invalidate subscription/provenance together. * Returns false when authoritative metadata arrived before this reset acquired @@ -602,13 +654,10 @@ export async function resetContextGraphReadinessForMissingMetadata(input: { if (!contextGraphId) return false; return withContextGraphReadinessMutationLock(input.agent, contextGraphId, async () => { - const locallyCurated = typeof input.agent.isCuratorOf === 'function' - ? await input.agent.isCuratorOf(contextGraphId).catch(() => false) - : false; - const hasConfirmedMeta = await input.agent.hasConfirmedMetaState( + const hasConfirmedMeta = await hasAuthoritativeContextGraphMetadata({ + agent: input.agent, contextGraphId, - { rejectUnregisteredPlaceholder: !locallyCurated }, - ).catch(() => false); + }); if (hasConfirmedMeta) return false; const patches = missingMetadataReadinessPatches(); diff --git a/packages/cli/src/daemon/catchup-status-response.ts b/packages/cli/src/daemon/catchup-status-response.ts new file mode 100644 index 000000000..88d4f0e41 --- /dev/null +++ b/packages/cli/src/daemon/catchup-status-response.ts @@ -0,0 +1,50 @@ +import type { + CatchupConvergenceStatus, + CatchupStatusResponse, +} from '../catchup-status-wire.js'; +import type { CatchupJob } from './types.js'; + +export type { + CatchupConvergenceStatus, + CatchupStatusResponse, +} from '../catchup-status-wire.js'; + +/** Project one immutable catch-up attempt plus live convergence onto the wire. */ +export function toCatchupStatusResponse( + job: CatchupJob, + convergence?: CatchupConvergenceStatus, +): CatchupStatusResponse { + const completedAfterAttempt = convergence?.state === 'complete' && + convergence.readinessUpdatedAt !== undefined && + job.finishedAt !== undefined && + convergence.readinessUpdatedAt > job.finishedAt && + (job.status === 'failed' || + job.status === 'deferred' || + job.status === 'unreachable'); + const invalidatedAfterAttempt = job.status === 'done' && + convergence !== undefined && + convergence.state !== 'complete'; + const status = completedAfterAttempt + ? 'done' + : invalidatedAfterAttempt + ? 'unreachable' + : job.status; + const { status: attemptStatus, error: attemptError, ...attemptFields } = job; + return { + ...attemptFields, + contextGraphId: job.contextGraphId, + includeSharedMemory: job.includeWorkspace, + status, + ...(status === attemptStatus && attemptError ? { error: attemptError } : {}), + ...(status !== attemptStatus + ? { + attempt: { + status: attemptStatus, + ...(attemptError ? { error: attemptError } : {}), + }, + } + : {}), + ...(convergence ? { convergence } : {}), + ...(completedAfterAttempt ? { completedAfterAttempt: true } : {}), + }; +} diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts new file mode 100644 index 000000000..42beb67d6 --- /dev/null +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -0,0 +1,325 @@ +import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg-node-ui'; +import type { CatchupRunner } from '../catchup-runner.js'; +import type { CatchupJobResult } from '../catchup-result-wire.js'; +import { + catchupResultHasCleanResponse, + classifyContextGraphCatchupReadiness, +} from '../context-graph-readiness.js'; +import type { + CatchupCoordinator, + CatchupJob, + CatchupTracker, +} from './types.js'; + +export interface ContextGraphCatchupCoordinatorEffects { + runner: Pick; + readReadiness: (contextGraphId: string) => ContextGraphReadinessProvenance; + hasConfirmedMeta: (contextGraphId: string) => Promise; + isPrivate: (contextGraphId: string) => Promise; + writeReadiness: ( + contextGraphId: string, + patch: { durableVerified: boolean; sharedMemoryVerified: boolean }, + ) => void; + markSubscriptionState: ( + contextGraphId: string, + patch: { + synced: boolean; + sharedMemorySynced: boolean; + metaSynced: boolean; + pendingMeta: boolean; + }, + ) => void; + emitProjectSynced: ( + contextGraphId: string, + payload: { + dataSynced: number; + sharedMemorySynced: number; + verifiedPrivateOnlyResponses: number; + }, + ) => void; + now?: () => number; + createJobId?: () => string; + trace?: (message: string) => void; +} + +export class ContextGraphCatchupCoordinatorService { + private readonly now: () => number; + private readonly createJobId: () => string; + private readonly inFlightByContextGraph: Map; + + constructor( + private readonly tracker: CatchupTracker, + private readonly effects: ContextGraphCatchupCoordinatorEffects, + ) { + this.now = effects.now ?? Date.now; + this.createJobId = effects.createJobId ?? (() => + `${this.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`); + this.inFlightByContextGraph = tracker.inFlightByContextGraph; + } + + /** Reuse active work while preserving each caller's immutable plane scope. */ + coalesceActive(input: { + contextGraphId: string; + includeSharedMemory: boolean; + }): CatchupJob | undefined { + const coordinator = this.inFlightByContextGraph.get(input.contextGraphId); + if (!coordinator) return undefined; + const baseJob = this.tracker.jobs.get(coordinator.baseJobId); + const upgradeJob = coordinator.upgradeJobId + ? this.tracker.jobs.get(coordinator.upgradeJobId) + : undefined; + const narrowProjectionJob = coordinator.narrowProjectionJobId + ? this.tracker.jobs.get(coordinator.narrowProjectionJobId) + : undefined; + const active = [baseJob, upgradeJob].some((candidate) => + candidate?.status === 'queued' || candidate?.status === 'running'); + if (!baseJob || !active) return undefined; + + if (!input.includeSharedMemory) { + if (!baseJob.includeWorkspace) return baseJob; + if (narrowProjectionJob) return narrowProjectionJob; + + const createdProjection = this.createJob(input.contextGraphId, false); + coordinator.narrowProjectionJobId = createdProjection.jobId; + this.tracker.latestByContextGraph.set( + input.contextGraphId, + createdProjection.jobId, + ); + return createdProjection; + } + + if (baseJob.includeWorkspace) return baseJob; + if (upgradeJob) return upgradeJob; + + const createdUpgrade = this.createJob(input.contextGraphId, true); + coordinator.upgradeJobId = createdUpgrade.jobId; + this.tracker.latestByContextGraph.set(input.contextGraphId, createdUpgrade.jobId); + return createdUpgrade; + } + + /** Start one detached serialized worker for a fresh per-CG catch-up. */ + start(input: { + contextGraphId: string; + includeSharedMemory: boolean; + readinessBeforeCatchup: ContextGraphReadinessProvenance; + }): CatchupJob { + const job = this.createJob(input.contextGraphId, input.includeSharedMemory); + this.tracker.latestByContextGraph.set(input.contextGraphId, job.jobId); + const coordinator: CatchupCoordinator = { + contextGraphId: input.contextGraphId, + baseJobId: job.jobId, + }; + this.inFlightByContextGraph.set(input.contextGraphId, coordinator); + this.pruneCompletedJobs(); + void this.run(coordinator, job, input.readinessBeforeCatchup); + return job; + } + + private createJob(contextGraphId: string, includeSharedMemory: boolean): CatchupJob { + const job: CatchupJob = { + jobId: this.createJobId(), + contextGraphId, + includeWorkspace: includeSharedMemory, + status: 'queued', + queuedAt: this.now(), + }; + this.tracker.jobs.set(job.jobId, job); + return job; + } + + private pruneCompletedJobs(): void { + const activeJobIds = new Set(); + for (const coordinator of this.inFlightByContextGraph.values()) { + activeJobIds.add(coordinator.baseJobId); + if (coordinator.upgradeJobId) activeJobIds.add(coordinator.upgradeJobId); + if (coordinator.narrowProjectionJobId) { + activeJobIds.add(coordinator.narrowProjectionJobId); + } + } + while (this.tracker.jobs.size > 100) { + let oldest: CatchupJob | undefined; + for (const candidate of this.tracker.jobs.values()) { + if (activeJobIds.has(candidate.jobId)) continue; + if (!oldest || candidate.queuedAt < oldest.queuedAt) oldest = candidate; + } + if (!oldest) break; + this.tracker.jobs.delete(oldest.jobId); + if ( + this.tracker.latestByContextGraph.get(oldest.contextGraphId) === oldest.jobId + ) { + this.tracker.latestByContextGraph.delete(oldest.contextGraphId); + } + } + } + + private async run( + coordinator: CatchupCoordinator, + baseJob: CatchupJob, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + let attemptJob = baseJob; + + try { + let readinessBeforeAttempt = readinessBeforeCatchup; + while (true) { + await this.runAttempt(attemptJob, readinessBeforeAttempt); + if (attemptJob === baseJob && baseJob.includeWorkspace) { + await this.settleNarrowProjection( + coordinator, + baseJob, + readinessBeforeAttempt, + ); + } + if ( + coordinator.upgradeJobId && + !attemptJob.includeWorkspace && + attemptJob.status !== 'denied' + ) { + const upgradeJob = this.tracker.jobs.get(coordinator.upgradeJobId); + if (!upgradeJob) break; + attemptJob = upgradeJob; + readinessBeforeAttempt = this.effects.readReadiness( + coordinator.contextGraphId, + ); + continue; + } + if (!attemptJob.includeWorkspace) { + this.settleQueuedJobFrom(coordinator.upgradeJobId, attemptJob); + } + break; + } + } catch (error) { + attemptJob.error = error instanceof Error ? error.message : String(error); + attemptJob.status = 'failed'; + attemptJob.finishedAt = this.now(); + this.settleQueuedJobFrom(coordinator.upgradeJobId, attemptJob); + this.settleQueuedJobFrom(coordinator.narrowProjectionJobId, attemptJob); + this.effects.trace?.( + `[catchup] job=${attemptJob.jobId} contextGraph=${coordinator.contextGraphId} threw: ${attemptJob.error}`, + ); + } finally { + attemptJob.finishedAt ??= this.now(); + if (this.inFlightByContextGraph.get(coordinator.contextGraphId) === coordinator) { + this.inFlightByContextGraph.delete(coordinator.contextGraphId); + } + } + } + + private async runAttempt( + job: CatchupJob, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + job.status = 'running'; + job.startedAt ??= this.now(); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} started`, + ); + const result = await this.effects.runner.run({ + contextGraphId: job.contextGraphId, + includeSharedMemory: job.includeWorkspace, + }); + await this.applyResult(job, result, readinessBeforeCatchup, true); + } + + private async applyResult( + job: CatchupJob, + result: CatchupJobResult, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + applyEffects: boolean, + ): Promise { + job.result = result; + job.error = undefined; + + if (result.deferredBackpressure > 0 && !result.denied) { + job.status = 'deferred'; + job.error = 'Sync deferred by local scheduler backpressure; retry when capacity is available.'; + } else { + const inspectReadiness = catchupResultHasCleanResponse(result); + const hasConfirmedMeta = inspectReadiness + ? await this.effects.hasConfirmedMeta(job.contextGraphId) + : false; + const isPrivate = hasConfirmedMeta + ? await this.effects.isPrivate(job.contextGraphId) + : false; + const classification = classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory: job.includeWorkspace, + hasConfirmedMeta, + isPrivate, + readinessBeforeCatchup, + }); + job.status = classification.jobStatus; + job.error = classification.error; + if (applyEffects && classification.readinessPatch) { + this.effects.writeReadiness(job.contextGraphId, classification.readinessPatch); + } + if (applyEffects && classification.statePatch) { + this.effects.markSubscriptionState( + job.contextGraphId, + classification.statePatch, + ); + } + if (applyEffects && classification.eventPayload) { + this.effects.emitProjectSynced( + job.contextGraphId, + classification.eventPayload, + ); + } + if (job.status === 'done' && result.deferredBackpressure > 0) { + job.status = 'deferred'; + job.error = 'Sync deferred by local scheduler backpressure; retry when capacity is available.'; + } + } + job.finishedAt = this.now(); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} status=${job.status} ` + + `peers=${result.peersTried}/${result.syncCapablePeers} ` + + `connected=${result.totalPeers ?? result.connectedPeers} ` + + `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, + ); + } + + private settleQueuedJobFrom( + targetJobId: string | undefined, + source: CatchupJob, + ): void { + const target = targetJobId ? this.tracker.jobs.get(targetJobId) : undefined; + if (!target || target.status !== 'queued') return; + target.status = source.status; + target.error = source.error; + target.result = source.result; + target.startedAt = source.startedAt; + target.finishedAt = this.now(); + } + + private async settleNarrowProjection( + coordinator: CatchupCoordinator, + source: CatchupJob, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + const projection = coordinator.narrowProjectionJobId + ? this.tracker.jobs.get(coordinator.narrowProjectionJobId) + : undefined; + if (!projection || projection.status !== 'queued') return; + + if (!source.result || source.status === 'failed') { + this.settleQueuedJobFrom(coordinator.narrowProjectionJobId, source); + return; + } + + projection.status = 'running'; + projection.startedAt = source.startedAt; + try { + await this.applyResult( + projection, + source.result, + readinessBeforeCatchup, + false, + ); + } catch (error) { + projection.status = 'failed'; + projection.error = error instanceof Error ? error.message : String(error); + projection.finishedAt = this.now(); + } + } +} diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index 54e636550..327405b48 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -146,7 +146,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from './types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 4760c8efe..1969c8a15 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -211,7 +211,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from './types.js'; import { type MarkItDownTarget, @@ -3352,6 +3351,7 @@ export async function runDaemonInner( const catchupTracker: CatchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), + inFlightByContextGraph: new Map(), }; // --- Extraction Pipelines --- diff --git a/packages/cli/src/daemon/routes/agent-chat.ts b/packages/cli/src/daemon/routes/agent-chat.ts index 44cdeb0e0..a24e469d7 100644 --- a/packages/cli/src/daemon/routes/agent-chat.ts +++ b/packages/cli/src/daemon/routes/agent-chat.ts @@ -166,7 +166,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index b43e852ec..fba019ae8 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -114,9 +114,8 @@ import { import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; import { - catchupResultHasCleanResponse, - classifyContextGraphCatchupReadiness, classifyExistingContextGraphReadiness, + hasAuthoritativeContextGraphMetadata, readContextGraphReadiness, writeContextGraphReadiness, } from '../../context-graph-readiness.js'; @@ -167,10 +166,9 @@ import { import { type CatchupJobState, type CatchupJob, - type CatchupCoordinator, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; +import { ContextGraphCatchupCoordinatorService } from '../context-graph-catchup-coordinator.js'; import { type MarkItDownTarget, manifestRepoRoot, @@ -1742,46 +1740,40 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise(); - const existingCoordinator = inFlightByContextGraph.get(contextGraphId); + const catchupCoordinator = new ContextGraphCatchupCoordinatorService( + catchupTracker, + { + runner: daemonState.catchupRunner!, + readReadiness: (id) => readContextGraphReadiness(dashDb, id), + hasConfirmedMeta: (id) => hasAuthoritativeContextGraphMetadata({ + agent, + contextGraphId: id, + }), + isPrivate: (id) => agent.isPrivateContextGraph(id).catch(() => true), + writeReadiness: (id, patch) => writeContextGraphReadiness(dashDb, id, patch), + markSubscriptionState: (id, patch) => + agent.markContextGraphSubscriptionState(id, patch), + emitProjectSynced: (id, payload) => { + agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { + contextGraphId: id, + ...payload, + }); + }, + ...(DEBUG_SYNC_TRACE + ? { trace: (message: string) => console.log(message) } + : {}), + }, + ); const existingJobId = catchupTracker.latestByContextGraph.get(contextGraphId); const existingJob = existingJobId ? catchupTracker.jobs.get(existingJobId) : undefined; let readinessBeforeCatchup = readContextGraphReadiness(dashDb, contextGraphId); if (existingSub?.subscribed) { - const baseJob = existingCoordinator - ? catchupTracker.jobs.get(existingCoordinator.baseJobId) - : undefined; - const upgradeJob = existingCoordinator?.upgradeJobId - ? catchupTracker.jobs.get(existingCoordinator.upgradeJobId) - : undefined; - const coordinatorActive = existingCoordinator && - [baseJob, upgradeJob].some((candidate) => - candidate?.status === 'queued' || candidate?.status === 'running'); - if (existingCoordinator && baseJob && coordinatorActive) { - // Coalesce onto one serialized worker without changing the contract of - // an already-issued jobId. A later VM+SWM request gets a separate - // public job record while the coordinator records the monotonic work - // upgrade for the running worker. - let responseJob = baseJob; - if (shouldSyncSharedMemory && !baseJob.includeWorkspace) { - existingCoordinator.requestedIncludeSharedMemory = true; - responseJob = upgradeJob ?? (() => { - const upgradeJobId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; - const createdUpgradeJob: CatchupJob = { - jobId: upgradeJobId, - contextGraphId, - includeWorkspace: true, - status: 'queued', - queuedAt: Date.now(), - }; - catchupTracker.jobs.set(upgradeJobId, createdUpgradeJob); - catchupTracker.latestByContextGraph.set(contextGraphId, upgradeJobId); - existingCoordinator.upgradeJobId = upgradeJobId; - return createdUpgradeJob; - })(); - } + const responseJob = catchupCoordinator.coalesceActive({ + contextGraphId, + includeSharedMemory: shouldSyncSharedMemory, + }); + if (responseJob) { return jsonResponse(res, 200, { subscribed: contextGraphId, syncMode: effectiveSyncMode, @@ -1859,177 +1851,11 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise 100) { - let oldestId: string | undefined; - let oldestQueuedAt = Number.POSITIVE_INFINITY; - for (const [id, entry] of catchupTracker.jobs.entries()) { - if (entry.queuedAt < oldestQueuedAt) { - oldestQueuedAt = entry.queuedAt; - oldestId = id; - } - } - if (!oldestId) break; - const removed = catchupTracker.jobs.get(oldestId); - catchupTracker.jobs.delete(oldestId); - if ( - removed && - catchupTracker.latestByContextGraph.get(removed.contextGraphId) === oldestId - ) { - catchupTracker.latestByContextGraph.delete(removed.contextGraphId); - } - } - - void (async () => { - let attemptJob = job; - const settleQueuedUpgradeFrom = (source: CatchupJob) => { - const queuedUpgrade = coordinator.upgradeJobId - ? catchupTracker.jobs.get(coordinator.upgradeJobId) - : undefined; - if (!queuedUpgrade || queuedUpgrade.status !== 'queued') return; - queuedUpgrade.status = source.status; - queuedUpgrade.error = source.error; - queuedUpgrade.result = source.result; - queuedUpgrade.startedAt = source.startedAt; - queuedUpgrade.finishedAt = Date.now(); - }; - try { - let attemptReadinessBeforeCatchup = readinessBeforeCatchup; - while (true) { - attemptJob.status = 'running'; - attemptJob.startedAt ??= Date.now(); - const attemptIncludeSharedMemory = attemptJob.includeWorkspace; - if (DEBUG_SYNC_TRACE) { - console.log( - `[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} started`, - ); - } - const result = await daemonState.catchupRunner!.run({ - contextGraphId: contextGraphId, - includeSharedMemory: attemptIncludeSharedMemory, - }); - attemptJob.result = result; - attemptJob.error = undefined; - // Local scheduler pressure cut the round short. An incomplete round has - // no readiness to inspect and must never finalize the subscription, so - // short-circuit the whole classification path and report a distinct - // retryable status. A remote denial still wins: waiting for local - // capacity will never clear it. - if (result.deferredBackpressure > 0 && !result.denied) { - attemptJob.status = "deferred"; - attemptJob.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} deferred by local scheduler: ${result.deferredBackpressure}`); - } else { - const inspectReadiness = catchupResultHasCleanResponse(result); - const hasConfirmedMeta = inspectReadiness - ? await agent.hasConfirmedMetaState(contextGraphId).catch(() => false) - : false; - const isPrivate = hasConfirmedMeta - ? await agent.isPrivateContextGraph(contextGraphId).catch(() => true) - : false; - const classification = classifyContextGraphCatchupReadiness({ - result, - includeSharedMemory: attemptIncludeSharedMemory, - hasConfirmedMeta, - isPrivate, - readinessBeforeCatchup: attemptReadinessBeforeCatchup, - }); - - attemptJob.status = classification.jobStatus; - attemptJob.error = classification.error; - if (classification.readinessPatch) { - writeContextGraphReadiness( - dashDb, - contextGraphId, - classification.readinessPatch, - ); - } - if (classification.statePatch) { - agent.markContextGraphSubscriptionState( - contextGraphId, - classification.statePatch, - ); - } - if (classification.eventPayload) { - agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { - contextGraphId, - ...classification.eventPayload, - }); - } - - // Denial took precedence above, but a denied-yet-still-deferred round - // is likewise incomplete: never let it settle as a successful "done". - if (attemptJob.status === "done" && result.deferredBackpressure > 0) { - attemptJob.status = "deferred"; - attemptJob.error = "Sync deferred by local scheduler backpressure; retry when capacity is available."; - } - } - attemptJob.finishedAt = Date.now(); - - if (DEBUG_SYNC_TRACE) { - if (attemptJob.status === 'denied') { - console.log(`[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} denied by remote peer(s): ${result.deniedPeers}`); - } - console.log( - `[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} status=${attemptJob.status} ` + - `peers=${result.peersTried}/${result.syncCapablePeers} ` + - `connected=${result.totalPeers ?? result.connectedPeers} ` + - `data=${result.dataSynced} swm=${result.sharedMemorySynced} denied=${result.denied}`, - ); - } - - // A caller may upgrade a running VM-only selection to VM+SWM. Finish - // the narrow public job, then satisfy the wider public job through - // this same serialized coordinator instead of launching competing - // per-CG catch-up work. - if ( - coordinator.requestedIncludeSharedMemory && - !attemptIncludeSharedMemory && - attemptJob.status !== 'denied' - ) { - const upgradeJob = coordinator.upgradeJobId - ? catchupTracker.jobs.get(coordinator.upgradeJobId) - : undefined; - if (!upgradeJob) break; - attemptJob = upgradeJob; - attemptReadinessBeforeCatchup = readContextGraphReadiness( - dashDb, - contextGraphId, - ); - continue; - } - if (!attemptIncludeSharedMemory) settleQueuedUpgradeFrom(attemptJob); - break; - } - } catch (err) { - attemptJob.error = err instanceof Error ? err.message : String(err); - attemptJob.status = "failed"; - attemptJob.finishedAt = Date.now(); - settleQueuedUpgradeFrom(attemptJob); - if (DEBUG_SYNC_TRACE) console.log(`[catchup] job=${attemptJob.jobId} contextGraph=${contextGraphId} threw: ${attemptJob.error}`); - } finally { - attemptJob.finishedAt ??= Date.now(); - if (inFlightByContextGraph.get(contextGraphId) === coordinator) { - inFlightByContextGraph.delete(contextGraphId); - } - } - })(); + includeSharedMemory: shouldSyncSharedMemory, + readinessBeforeCatchup, + }); return jsonResponse(res, 200, { subscribed: contextGraphId, @@ -2037,7 +1863,7 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise { } const subscription = agent.getSubscribedContextGraphs().get(job.contextGraphId); - const hasConfirmedMeta = await agent.hasConfirmedMetaState(job.contextGraphId) - .catch(() => false); + const hasConfirmedMeta = await hasAuthoritativeContextGraphMetadata({ + agent, + contextGraphId: job.contextGraphId, + }); const convergence = { ...describeContextGraphConvergence({ readiness: readContextGraphReadiness(dashDb, job.contextGraphId), diff --git a/packages/cli/src/daemon/routes/status.ts b/packages/cli/src/daemon/routes/status.ts index 83b1d2644..5fcb03953 100644 --- a/packages/cli/src/daemon/routes/status.ts +++ b/packages/cli/src/daemon/routes/status.ts @@ -160,7 +160,6 @@ import { type CatchupJobState, type CatchupJob, type CatchupTracker, - toCatchupStatusResponse, } from '../types.js'; import { type MarkItDownTarget, diff --git a/packages/cli/src/daemon/types.ts b/packages/cli/src/daemon/types.ts index e3d3a00d2..90babcd4e 100644 --- a/packages/cli/src/daemon/types.ts +++ b/packages/cli/src/daemon/types.ts @@ -3,25 +3,9 @@ // Pure type/interface declarations used across the daemon sub-modules. import type { CatchupJobResult } from '../catchup-runner.js'; -import type { ContextGraphConvergenceSnapshot } from '../context-graph-readiness.js'; +import type { CatchupJobState } from '../catchup-status-wire.js'; -export type CatchupJobState = - | "queued" - | "running" - | "done" - | "failed" - | "denied" - /** Local scheduler capacity was unavailable; retry is safe. */ - | "deferred" - /** - * Catchup completed but no peer could deliver the CG content within - * the run — every per-peer sync round either failed or returned - * nothing while no responder explicitly denied access. Distinct from - * `denied` (curator refused) and `failed` (the worker itself threw) - * so the UI can render targeted copy + a "send signed join request" - * CTA without misclassifying slow public CGs as denied. - */ - | "unreachable"; +export type { CatchupJobState } from '../catchup-status-wire.js'; export interface CatchupJob { jobId: string; @@ -43,42 +27,12 @@ export interface CatchupJob { export interface CatchupCoordinator { contextGraphId: string; baseJobId: string; - requestedIncludeSharedMemory: boolean; upgradeJobId?: string; + narrowProjectionJobId?: string; } export interface CatchupTracker { jobs: Map; latestByContextGraph: Map; - inFlightByContextGraph?: Map; -} - -export interface CatchupConvergenceStatus extends ContextGraphConvergenceSnapshot { - syncMode: 'on-demand' | 'always-on'; - automaticRetryActive: boolean; -} - -export function toCatchupStatusResponse( - job: CatchupJob, - convergence?: CatchupConvergenceStatus, -) { - const completedAfterAttempt = convergence?.state === 'complete' && - (job.status === 'failed' || - job.status === 'deferred' || - job.status === 'unreachable'); - return { - ...job, - contextGraphId: job.contextGraphId, - includeSharedMemory: job.includeWorkspace, - attemptStatus: job.status, - ...(completedAfterAttempt - ? { - status: 'done' as const, - error: undefined, - ...(job.error ? { attemptError: job.error } : {}), - } - : {}), - ...(convergence ? { convergence } : {}), - ...(completedAfterAttempt ? { completedAfterAttempt: true } : {}), - }; + inFlightByContextGraph: Map; } diff --git a/packages/cli/test/catchup-status-cli.test.ts b/packages/cli/test/catchup-status-cli.test.ts new file mode 100644 index 000000000..8f4a509c3 --- /dev/null +++ b/packages/cli/test/catchup-status-cli.test.ts @@ -0,0 +1,64 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ApiClient } from '../src/api-client.js'; +import type { CatchupStatusResponse } from '../src/catchup-status-wire.js'; +import { + printCatchupStatus, + runCatchupStatusCommand, +} from '../src/cli-helpers.js'; + +const recoveredStatus: CatchupStatusResponse = { + jobId: 'job-recovered', + contextGraphId: 'cg-recovered', + includeWorkspace: true, + includeSharedMemory: true, + status: 'done', + queuedAt: 1, + startedAt: 2, + finishedAt: 3, + attempt: { + status: 'unreachable', + error: 'durable VM missing', + }, + convergence: { + state: 'complete', + required: { metadata: true, durable: true, sharedMemory: true }, + verified: { metadata: true, durable: true, sharedMemory: true }, + missing: [], + readinessUpdatedAt: 4, + observedAt: 5, + syncMode: 'on-demand', + automaticRetryActive: true, + }, + completedAfterAttempt: true, +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('catch-up status CLI', () => { + it('renders actionable recovery with historical attempt diagnostics', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); + + printCatchupStatus(recoveredStatus); + + const lines = log.mock.calls.map(([line]) => String(line)); + expect(lines).toContain('Status: done'); + expect(lines).toContain('Last attempt: unreachable'); + expect(lines).toContain('Attempt error: durable VM missing'); + expect(lines).toContain('Convergence: complete'); + expect(lines).toContain('Recovered: a later synchronization completed the selected graph'); + }); + + it('stops watch mode on the canonical actionable status', async () => { + const catchupStatus = vi.fn().mockResolvedValue(recoveredStatus); + vi.spyOn(ApiClient, 'connect').mockResolvedValue({ catchupStatus } as ApiClient); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'clear').mockImplementation(() => {}); + + await runCatchupStatusCommand('cg-recovered', { watch: true, interval: 1 }); + + expect(catchupStatus).toHaveBeenCalledTimes(1); + expect(catchupStatus).toHaveBeenCalledWith('cg-recovered'); + }); +}); diff --git a/packages/cli/test/catchup-status-convergence-route.test.ts b/packages/cli/test/catchup-status-convergence-route.test.ts new file mode 100644 index 000000000..8f698c67c --- /dev/null +++ b/packages/cli/test/catchup-status-convergence-route.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; +import { privateSharedMemoryOnlyResult } from './helpers/context-graph-catchup-fixtures.js'; +import { ContextGraphSubscribeRouteHarness } from './helpers/context-graph-subscribe-route-harness.js'; + +async function runIncompletePrivateAttempt( + options: { strictHasConfirmedMeta?: boolean; locallyCurated?: boolean } = {}, +) { + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + strictHasConfirmedMeta: options.strictHasConfirmedMeta, + locallyCurated: options.locallyCurated, + isPrivate: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: () => privateSharedMemoryOnlyResult(), + }); + const response = await harness.postSubscribe(); + const jobId = response.body.catchup.jobId as string; + const job = await harness.waitForJob(jobId); + return { harness, jobId, job }; +} + +describe('catch-up status live convergence', () => { + it('keeps a failed attempt failed when complete readiness predates the job', async () => { + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: true, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + readiness: { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: 1, + }, + runner: () => { + throw new Error('foreground attempt failed'); + }, + }); + try { + const response = await harness.postSubscribe(); + const jobId = response.body.catchup.jobId as string; + const job = await harness.waitForJob(jobId); + expect(job?.status).toBe('failed'); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'failed', + error: 'foreground attempt failed', + convergence: { + state: 'complete', + readinessUpdatedAt: 1, + }, + }); + } finally { + await harness.close(); + } + }); + + it('reports completion after persisted readiness recovers a failed attempt', async () => { + const { harness, jobId, job } = await runIncompletePrivateAttempt(); + try { + expect(job?.status).toBe('unreachable'); + harness.setCompleteReadiness((job?.finishedAt ?? Date.now()) + 1); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'done', + attempt: { + status: 'unreachable', + error: expect.stringContaining('durable VM'), + }, + completedAfterAttempt: true, + convergence: { + state: 'complete', + verified: { + metadata: true, + durable: true, + sharedMemory: true, + }, + missing: [], + }, + }); + } finally { + await harness.close(); + } + }); + + it('rejects legacy placeholder metadata instead of rewriting failure to done', async () => { + const { harness, jobId, job } = await runIncompletePrivateAttempt({ + strictHasConfirmedMeta: false, + locallyCurated: false, + }); + try { + expect(job?.status).toBe('unreachable'); + harness.setCompleteReadiness((job?.finishedAt ?? Date.now()) + 1); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'unreachable', + convergence: { + state: 'pending', + verified: { + metadata: false, + durable: false, + sharedMemory: false, + }, + missing: ['metadata', 'durable', 'sharedMemory'], + }, + }); + expect(harness.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: true, + }); + } finally { + await harness.close(); + } + }); + + it('preserves the local-curator metadata exception used by bootstrap', async () => { + const { harness, jobId, job } = await runIncompletePrivateAttempt({ + strictHasConfirmedMeta: false, + locallyCurated: true, + }); + try { + harness.setCompleteReadiness((job?.finishedAt ?? Date.now()) + 1); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'done', + completedAfterAttempt: true, + convergence: { state: 'complete', missing: [] }, + }); + expect(harness.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: false, + }); + } finally { + await harness.close(); + } + }); +}); diff --git a/packages/cli/test/catchup-status-response.test.ts b/packages/cli/test/catchup-status-response.test.ts index 0e4953dfd..7c008846e 100644 --- a/packages/cli/test/catchup-status-response.test.ts +++ b/packages/cli/test/catchup-status-response.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it } from 'vitest'; import { toCatchupStatusResponse, type CatchupConvergenceStatus, - type CatchupJob, -} from '../src/daemon/types.js'; +} from '../src/daemon/catchup-status-response.js'; +import type { CatchupJob } from '../src/daemon/types.js'; const completeConvergence: CatchupConvergenceStatus = { state: 'complete', @@ -41,20 +41,51 @@ describe('catch-up status response', () => { it('reports live completion while preserving a failed attempt as diagnostics', () => { expect(toCatchupStatusResponse(job('failed'), completeConvergence)).toMatchObject({ status: 'done', - attemptStatus: 'failed', - attemptError: 'failed attempt', - error: undefined, + attempt: { + status: 'failed', + error: 'failed attempt', + }, completedAfterAttempt: true, convergence: completeConvergence, }); + expect(toCatchupStatusResponse(job('failed'), completeConvergence)) + .not.toHaveProperty('error'); + }); + + it('does not hide a failed attempt behind readiness that predates it', () => { + const staleConvergence = { ...completeConvergence, readinessUpdatedAt: 3 }; + + expect(toCatchupStatusResponse(job('failed'), staleConvergence)).toMatchObject({ + status: 'failed', + error: 'failed attempt', + convergence: staleConvergence, + }); + expect(toCatchupStatusResponse(job('failed'), staleConvergence)) + .not.toHaveProperty('completedAfterAttempt'); + expect(toCatchupStatusResponse(job('failed'), staleConvergence)) + .not.toHaveProperty('attempt'); }); it('never lets historical readiness override a current authorization denial', () => { expect(toCatchupStatusResponse(job('denied'), completeConvergence)).toMatchObject({ status: 'denied', - attemptStatus: 'denied', error: 'denied attempt', convergence: completeConvergence, }); }); + + it('downgrades the actionable status when a completed attempt loses convergence', () => { + const invalidatedConvergence = { + ...completeConvergence, + state: 'pending' as const, + verified: { metadata: false, durable: false, sharedMemory: false }, + missing: ['metadata', 'durable', 'sharedMemory'] as const, + }; + + expect(toCatchupStatusResponse(job('done'), invalidatedConvergence)).toMatchObject({ + status: 'unreachable', + attempt: { status: 'done' }, + convergence: invalidatedConvergence, + }); + }); }); diff --git a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts new file mode 100644 index 000000000..90ba289cf --- /dev/null +++ b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it } from 'vitest'; +import { + privateDataOnlyResult, + publicDurableAndSharedMemoryResult, +} from './helpers/context-graph-catchup-fixtures.js'; +import { ContextGraphSubscribeRouteHarness } from './helpers/context-graph-subscribe-route-harness.js'; + +function deferred(): { + promise: Promise; + resolve: () => void; +} { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +describe('context graph catch-up route coalescing', () => { + it('serially upgrades an active VM-only request with a distinct VM plus SWM job', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async (_request, callNumber) => { + if (callNumber === 1) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return privateDataOnlyResult(); + } + return publicDurableAndSharedMemoryResult(); + }, + }); + + try { + const base = await harness.postSubscribe({ includeSharedMemory: false }); + await firstRunStarted.promise; + const upgrade = await harness.postSubscribe({ includeSharedMemory: true }); + + expect(upgrade.body.catchup).toMatchObject({ + status: 'queued', + includeWorkspace: true, + }); + expect(upgrade.body.catchup.jobId).not.toBe(base.body.catchup.jobId); + expect(harness.runCalls).toBe(1); + expect(harness.getJob(upgrade.body.catchup.jobId)?.status).toBe('queued'); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: upgrade.body.catchup.jobId, + status: 'queued', + includeWorkspace: true, + includeSharedMemory: true, + }); + + releaseFirstRun.resolve(); + const baseJob = await harness.waitForJob(base.body.catchup.jobId); + const upgradeJob = await harness.waitForJob(upgrade.body.catchup.jobId); + + expect(harness.runRequests).toEqual([ + { + contextGraphId: harness.contextGraphId, + includeSharedMemory: false, + }, + { + contextGraphId: harness.contextGraphId, + includeSharedMemory: true, + }, + ]); + expect(baseJob).toMatchObject({ includeWorkspace: false, status: 'done' }); + expect(upgradeJob).toMatchObject({ includeWorkspace: true, status: 'done' }); + await expect(harness.getStatus(base.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + await expect(harness.getStatus(upgrade.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: true }, + missing: [], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('projects broad in-flight work onto a distinct VM-only contract', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async () => { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return privateDataOnlyResult(); + }, + }); + + try { + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await firstRunStarted.promise; + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + + expect(narrow.body.catchup).toMatchObject({ + status: 'queued', + includeWorkspace: false, + }); + expect(narrow.body.catchup.jobId).not.toBe(broad.body.catchup.jobId); + expect(harness.runCalls).toBe(1); + + releaseFirstRun.resolve(); + const broadJob = await harness.waitForJob(broad.body.catchup.jobId); + const narrowJob = await harness.waitForJob(narrow.body.catchup.jobId); + + expect(harness.runCalls).toBe(1); + expect(broadJob).toMatchObject({ + includeWorkspace: true, + status: 'unreachable', + }); + expect(narrowJob).toMatchObject({ + includeWorkspace: false, + status: 'done', + }); + await expect(harness.getStatus(narrow.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + await expect(harness.getStatus(broad.body.catchup.jobId)).resolves.toMatchObject({ + status: 'unreachable', + convergence: { + state: 'partial', + required: { sharedMemory: true }, + missing: ['sharedMemory'], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('keeps VM-only success stable when the serialized SWM upgrade fails', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async (_request, callNumber) => { + if (callNumber === 1) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + } + return privateDataOnlyResult(); + }, + }); + + try { + const base = await harness.postSubscribe({ includeSharedMemory: false }); + await firstRunStarted.promise; + const upgrade = await harness.postSubscribe({ includeSharedMemory: true }); + releaseFirstRun.resolve(); + + const baseJob = await harness.waitForJob(base.body.catchup.jobId); + const upgradeJob = await harness.waitForJob(upgrade.body.catchup.jobId); + expect(baseJob).toMatchObject({ includeWorkspace: false, status: 'done' }); + expect(upgradeJob).toMatchObject({ + includeWorkspace: true, + status: 'unreachable', + error: expect.stringContaining('requested data plane'), + }); + await expect(harness.getStatus(base.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + await expect(harness.getStatus(upgrade.body.catchup.jobId)).resolves.toMatchObject({ + status: 'unreachable', + convergence: { + state: 'partial', + required: { sharedMemory: true }, + missing: ['sharedMemory'], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); +}); diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts new file mode 100644 index 000000000..cd3ade503 --- /dev/null +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -0,0 +1,230 @@ +import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg-node-ui'; +import { describe, expect, it, vi } from 'vitest'; +import { ContextGraphCatchupCoordinatorService } from '../src/daemon/context-graph-catchup-coordinator.js'; +import type { CatchupJob, CatchupTracker } from '../src/daemon/types.js'; +import { + cleanEmptyResult, + privateDataOnlyResult, + publicDurableAndSharedMemoryResult, +} from './helpers/context-graph-catchup-fixtures.js'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function waitForJob(job: CatchupJob): Promise { + for (let attempt = 0; attempt < 50 && !job.finishedAt; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + if (!job.finishedAt) throw new Error(`catch-up job ${job.jobId} did not settle`); +} + +function deniedResult() { + const result = cleanEmptyResult(); + result.denied = true; + result.deniedPeers = 1; + result.peersSucceeded = 0; + if (!result.cleanPlaneCompletions || !result.diagnostics) { + throw new Error('completion evidence missing'); + } + result.cleanPlaneCompletions.durable.emptyPeers = 0; + result.cleanPlaneCompletions.sharedMemory.emptyPeers = 0; + result.diagnostics.durable.emptyResponses = 0; + result.diagnostics.sharedMemory.emptyResponses = 0; + return result; +} + +function coordinatorFixture(options: { + failUpgrade?: boolean; + baseOutcome?: 'success' | 'throw' | 'denied'; +} = {}) { + const tracker: CatchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + inFlightByContextGraph: new Map(), + }; + let readiness: ContextGraphReadinessProvenance = { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }; + let sequence = 0; + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const run = vi.fn(async (request: { includeSharedMemory: boolean }) => { + if (!request.includeSharedMemory) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + if (options.baseOutcome === 'throw') { + throw new Error('base attempt failed'); + } + if (options.baseOutcome === 'denied') return deniedResult(); + return privateDataOnlyResult(); + } + return options.failUpgrade + ? privateDataOnlyResult() + : publicDurableAndSharedMemoryResult(); + }); + const service = new ContextGraphCatchupCoordinatorService(tracker, { + runner: { run }, + readReadiness: () => readiness, + hasConfirmedMeta: async () => true, + isPrivate: async () => false, + writeReadiness: (_contextGraphId, patch) => { + readiness = { ...readiness, ...patch, updatedAt: readiness.updatedAt + 1 }; + }, + markSubscriptionState: vi.fn(), + emitProjectSynced: vi.fn(), + createJobId: () => `job-${++sequence}`, + }); + return { + tracker, + service, + run, + firstRunStarted, + releaseFirstRun, + }; +} + +describe('ContextGraphCatchupCoordinatorService', () => { + it('keeps job scopes immutable and serializes one wider upgrade', async () => { + const fixture = coordinatorFixture(); + const base = fixture.service.start({ + contextGraphId: 'cg:one', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:one', + includeSharedMemory: true, + }); + const repeatedUpgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:one', + includeSharedMemory: true, + }); + + expect(base).toMatchObject({ jobId: 'job-1', includeWorkspace: false }); + expect(upgrade).toMatchObject({ jobId: 'job-2', includeWorkspace: true }); + expect(repeatedUpgrade?.jobId).toBe(upgrade?.jobId); + expect(fixture.run).toHaveBeenCalledTimes(1); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(fixture.run.mock.calls.map(([request]) => request)).toEqual([ + { contextGraphId: 'cg:one', includeSharedMemory: false }, + { contextGraphId: 'cg:one', includeSharedMemory: true }, + ]); + expect(base).toMatchObject({ includeWorkspace: false, status: 'done' }); + expect(upgrade).toMatchObject({ includeWorkspace: true, status: 'done' }); + expect(fixture.tracker.inFlightByContextGraph.has('cg:one')).toBe(false); + }); + + it('does not retroactively fail VM-only success when the wider upgrade is incomplete', async () => { + const fixture = coordinatorFixture({ failUpgrade: true }); + const base = fixture.service.start({ + contextGraphId: 'cg:two', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:two', + includeSharedMemory: true, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(base).toMatchObject({ includeWorkspace: false, status: 'done' }); + expect(upgrade).toMatchObject({ + includeWorkspace: true, + status: 'unreachable', + error: expect.stringContaining('requested data plane'), + }); + }); + + it('settles a queued wider job when the base attempt throws', async () => { + const fixture = coordinatorFixture({ baseOutcome: 'throw' }); + const base = fixture.service.start({ + contextGraphId: 'cg:failed-base', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:failed-base', + includeSharedMemory: true, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(fixture.run).toHaveBeenCalledTimes(1); + expect(base).toMatchObject({ status: 'failed', error: 'base attempt failed' }); + expect(upgrade).toMatchObject({ + status: 'failed', + error: 'base attempt failed', + finishedAt: expect.any(Number), + }); + }); + + it('settles a queued wider job without a second run when the base is denied', async () => { + const fixture = coordinatorFixture({ baseOutcome: 'denied' }); + const base = fixture.service.start({ + contextGraphId: 'cg:denied-base', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const upgrade = fixture.service.coalesceActive({ + contextGraphId: 'cg:denied-base', + includeSharedMemory: true, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(base); + if (!upgrade) throw new Error('upgrade job missing'); + await waitForJob(upgrade); + + expect(fixture.run).toHaveBeenCalledTimes(1); + expect(base.status).toBe('denied'); + expect(upgrade).toMatchObject({ + status: 'denied', + finishedAt: expect.any(Number), + }); + }); +}); diff --git a/packages/cli/test/context-graph-catchup-readiness.test.ts b/packages/cli/test/context-graph-catchup-readiness.test.ts index 9d27875c4..cd0e070df 100644 --- a/packages/cli/test/context-graph-catchup-readiness.test.ts +++ b/packages/cli/test/context-graph-catchup-readiness.test.ts @@ -3,6 +3,7 @@ import type { CatchupJobResult } from '../src/catchup-runner.js'; import { CONTEXT_GRAPH_READINESS_VERSION, classifyContextGraphCatchupReadiness, + combineCatchupPlaneEvidence, describeContextGraphConvergence, } from '../src/context-graph-readiness.js'; @@ -534,6 +535,36 @@ describe('context graph catch-up readiness classification', () => { }); describe('selected context-graph convergence snapshot', () => { + it('combines this-run evidence without reviving stale persisted proof', () => { + const planes = combineCatchupPlaneEvidence({ + readinessBeforeCatchup: { + version: 0, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt: 42, + }, + durableReadyThisRun: true, + sharedMemoryReadyThisRun: false, + includeSharedMemory: true, + hasConfirmedMeta: true, + }); + + expect(planes).toEqual({ + state: 'partial', + required: { + metadata: true, + durable: true, + sharedMemory: true, + }, + verified: { + metadata: true, + durable: true, + sharedMemory: false, + }, + missing: ['sharedMemory'], + }); + }); + it('requires independently verified VM and requested SWM planes', () => { const snapshot = describeContextGraphConvergence({ readiness: { diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index c28726590..f5c88d209 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -1,390 +1,14 @@ -import { afterEach, describe, expect, it } from 'vitest'; -import { createServer, type Server } from 'node:http'; -import type { CatchupJobResult, CatchupRunRequest } from '../src/catchup-runner.js'; -import { handleContextGraphRoutes } from '../src/daemon/routes/context-graph.js'; -import { handleQueryRoutes } from '../src/daemon/routes/query.js'; -import { daemonState } from '../src/daemon/state.js'; - -function cleanEmptyResult(): CatchupJobResult { - return { - connectedPeers: 1, - totalPeers: 1, - selectedPeers: 1, - syncCapablePeers: 1, - peersTried: 1, - peersResponded: 1, - peersSucceeded: 1, - dataSynced: 0, - sharedMemorySynced: 0, - denied: false, - deniedPeers: 0, - cleanPlaneCompletions: { - durable: { verifiedDataPeers: 0, emptyPeers: 1 }, - sharedMemory: { verifiedDataPeers: 0, emptyPeers: 1 }, - }, - diagnostics: { - noProtocolPeers: 0, - durable: { - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 0, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 1, - metaOnlyResponses: 0, - dataRejectedMissingMeta: 0, - rejectedKcs: 0, - failedPeers: 0, - failedPhases: 0, - }, - sharedMemory: { - fetchedMetaTriples: 0, - fetchedDataTriples: 0, - insertedMetaTriples: 0, - insertedDataTriples: 0, - bytesReceived: 0, - resumedPhases: 0, - timedOutPhases: 0, - completedPhases: 2, - checkpointAdvances: 0, - emptyResponses: 1, - droppedDataTriples: 0, - failedPeers: 0, - failedPhases: 0, - }, - }, - }; -} - -function privateMetaOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable) throw new Error('durable diagnostics missing'); - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedMetaTriples = 7; - result.diagnostics.durable.insertedMetaTriples = 1; - result.diagnostics.durable.metaOnlyResponses = 1; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.durable.emptyPeers = 0; - return result; -} - -function privateDataOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { - throw new Error('catch-up diagnostics missing'); - } - result.dataSynced = 3; - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedDataTriples = 3; - result.diagnostics.durable.insertedDataTriples = 3; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.completedPhases = 0; - result.diagnostics.sharedMemory.timedOutPhases = 1; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 0, emptyPeers: 0 }; - return result; -} - -function privateSharedMemoryOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.sharedMemory) { - throw new Error('shared-memory diagnostics missing'); - } - result.sharedMemorySynced = 4; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.fetchedDataTriples = 4; - result.diagnostics.sharedMemory.insertedDataTriples = 4; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; - return result; -} - -function publicDurableAndSharedMemoryResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { - throw new Error('catch-up diagnostics missing'); - } - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.dataSynced = 3; - result.sharedMemorySynced = 4; - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedDataTriples = 3; - result.diagnostics.durable.insertedDataTriples = 3; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.fetchedDataTriples = 4; - result.diagnostics.sharedMemory.insertedDataTriples = 4; - result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; - return result; -} +import { describe, expect, it } from 'vitest'; +import { + cleanEmptyResult, + privateDataOnlyResult, + privateMetaOnlyResult, + privateSharedMemoryOnlyResult, + publicDurableAndSharedMemoryResult, +} from './helpers/context-graph-catchup-fixtures.js'; +import { runSubscribeScenario as subscribe } from './helpers/context-graph-subscribe-route-harness.js'; describe('context graph subscribe readiness requires authoritative metadata', () => { - const previousCatchupRunner = daemonState.catchupRunner; - let server: Server | undefined; - - afterEach(async () => { - daemonState.catchupRunner = previousCatchupRunner; - if (!server) return; - await new Promise((resolve, reject) => { - server!.close((err) => (err ? reject(err) : resolve())); - }); - server = undefined; - }); - - async function subscribe(opts: { - initial?: Record; - hasConfirmedMeta: boolean; - hasConfirmedMetaAfterCatchup?: boolean; - isPrivate?: boolean; - allowedAgents?: string[]; - callerAddress?: string; - result?: CatchupJobResult; - includeSharedMemory?: boolean; - syncMode?: unknown; - coalescedUpgradeResult?: CatchupJobResult; - simulateAutomaticRecovery?: boolean; - readiness?: { - version: number; - durableVerified: boolean; - sharedMemoryVerified: boolean; - updatedAt?: number; - }; - }): Promise<{ - response: any; - responseStatus: number; - job: any; - runCalls: number; - runRequests: CatchupRunRequest[]; - subscribeCalls: Array<{ - id: string; - options: { syncMode?: 'on-demand' | 'always-on' } | undefined; - }>; - state: Record; - patches: Array>; - readiness: Record | undefined; - statusResponse: any; - coalescedResponse?: any; - coalescedJob?: any; - coalescedStatusResponse?: any; - }> { - const contextGraphId = `readiness-${Math.random().toString(36).slice(2, 8)}`; - const state = new Map>(); - if (opts.initial) state.set(contextGraphId, { ...opts.initial }); - const patches: Array> = []; - const catchupTracker = { - jobs: new Map(), - latestByContextGraph: new Map(), - }; - let runCalls = 0; - const runRequests: CatchupRunRequest[] = []; - const subscribeCalls: Array<{ - id: string; - options: { syncMode?: 'on-demand' | 'always-on' } | undefined; - }> = []; - let readiness = opts.readiness - ? { ...opts.readiness, updatedAt: opts.readiness.updatedAt ?? Date.now() } - : undefined; - let releaseFirstRun: (() => void) | undefined; - let noteFirstRunStarted: (() => void) | undefined; - const firstRunGate = opts.coalescedUpgradeResult - ? new Promise((resolve) => { releaseFirstRun = resolve; }) - : undefined; - const firstRunStarted = opts.coalescedUpgradeResult - ? new Promise((resolve) => { noteFirstRunStarted = resolve; }) - : undefined; - - daemonState.catchupRunner = { - run: async (request) => { - runCalls += 1; - const callNumber = runCalls; - runRequests.push(request); - if (callNumber === 1 && firstRunGate) { - noteFirstRunStarted?.(); - await firstRunGate; - } - return callNumber === 2 && opts.coalescedUpgradeResult - ? opts.coalescedUpgradeResult - : opts.result ?? cleanEmptyResult(); - }, - close: async () => {}, - }; - - const agent = { - getContextGraphAllowedAgents: async () => opts.allowedAgents ?? [], - getSubscribedContextGraphs: () => state, - subscribeToContextGraph: ( - id: string, - options?: { syncMode?: 'on-demand' | 'always-on' }, - ) => { - subscribeCalls.push({ id, options }); - const previous = state.get(id); - const effectiveSyncMode = previous?.subscribed && previous.syncMode === 'always-on' - ? 'always-on' - : options?.syncMode ?? previous?.syncMode ?? 'always-on'; - const applied = { - ...previous, - subscribed: true, - synced: previous?.synced ?? false, - syncMode: effectiveSyncMode, - }; - state.set(id, applied); - return applied; - }, - markContextGraphSubscriptionState: (id: string, patch: Record) => { - patches.push({ ...patch }); - state.set(id, { ...state.get(id), ...patch }); - }, - hasConfirmedMetaState: async () => { - return runCalls > 0 - ? opts.hasConfirmedMetaAfterCatchup ?? opts.hasConfirmedMeta - : opts.hasConfirmedMeta; - }, - isPrivateContextGraph: async () => opts.isPrivate ?? false, - resolveAgentByToken: () => undefined, - getDefaultAgentAddress: () => opts.callerAddress ?? '0x0000000000000000000000000000000000000001', - }; - - server = createServer(async (req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); - const routeContext = { - req, - res, - agent, - publisherControl: {}, - publisherRuntime: null, - config: { auth: { enabled: false } }, - startedAt: Date.now(), - dashDb: { - getContextGraphReadinessProvenance: () => readiness ?? null, - setContextGraphReadinessProvenance: (_id: string, next: Record) => { - readiness = { ...next, updatedAt: Date.now() } as typeof readiness; - }, - }, - opWallets: {}, - network: {}, - tracker: {}, - memoryManager: {}, - bridgeAuthToken: undefined, - nodeVersion: 'test', - nodeCommit: 'test', - catchupTracker, - extractionRegistry: {}, - fileStore: {}, - extractionStatus: new Map(), - assertionImportLocks: new Map(), - vectorStore: {}, - embeddingProvider: null, - validTokens: new Set(), - apiHost: '127.0.0.1', - apiPortRef: { value: 0 }, - routePlugins: [], - url, - path: url.pathname, - requestToken: undefined, - requestAgentAddress: undefined, - } as any; - await handleContextGraphRoutes(routeContext); - if (!res.writableEnded) await handleQueryRoutes(routeContext); - if (!res.writableEnded) { - res.statusCode = 404; - res.end(); - } - }); - await new Promise((resolve) => server!.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (!address || typeof address === 'string') throw new Error('route test server did not bind'); - - const httpResponse = await fetch(`http://127.0.0.1:${address.port}/api/context-graph/subscribe`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - contextGraphId, - includeSharedMemory: opts.includeSharedMemory ?? true, - ...(opts.syncMode !== undefined ? { syncMode: opts.syncMode } : {}), - }), - }); - const response = await httpResponse.json() as any; - const jobId = response.catchup?.jobId as string | undefined; - - let coalescedResponse: any; - if (firstRunStarted && releaseFirstRun) { - await firstRunStarted; - coalescedResponse = await fetch( - `http://127.0.0.1:${address.port}/api/context-graph/subscribe`, - { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - contextGraphId, - includeSharedMemory: true, - ...(opts.syncMode !== undefined ? { syncMode: opts.syncMode } : {}), - }), - }, - ).then((result) => result.json()); - releaseFirstRun(); - } - - const coalescedJobId = coalescedResponse?.catchup?.jobId as string | undefined; - - for (let i = 0; jobId && i < 50; i++) { - const originalFinished = catchupTracker.jobs.get(jobId)?.finishedAt; - const coalescedFinished = !coalescedJobId || - catchupTracker.jobs.get(coalescedJobId)?.finishedAt; - if (originalFinished && coalescedFinished) break; - await new Promise((resolve) => setTimeout(resolve, 5)); - } - - if (opts.simulateAutomaticRecovery) { - readiness = { - version: 1, - durableVerified: true, - sharedMemoryVerified: true, - updatedAt: Date.now(), - }; - state.set(contextGraphId, { - ...state.get(contextGraphId), - synced: true, - sharedMemorySynced: true, - metaSynced: true, - pendingMeta: false, - }); - } - - const statusResponse = jobId - ? await fetch( - `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, - ).then((result) => result.json()) - : null; - const coalescedStatusResponse = coalescedJobId - ? await fetch( - `http://127.0.0.1:${address.port}/api/sync/catchup-status?jobId=${encodeURIComponent(coalescedJobId)}`, - ).then((result) => result.json()) - : null; - - return { - response, - responseStatus: httpResponse.status, - job: jobId ? catchupTracker.jobs.get(jobId) : undefined, - runCalls, - runRequests, - subscribeCalls, - state: state.get(contextGraphId) ?? {}, - patches, - readiness, - statusResponse, - coalescedResponse, - coalescedJob: coalescedJobId - ? catchupTracker.jobs.get(coalescedJobId) - : undefined, - coalescedStatusResponse, - }; - } - it('keeps omitted sync mode backward-compatible as always-on', async () => { const result = await subscribe({ hasConfirmedMeta: false, @@ -464,6 +88,9 @@ describe('context graph subscribe readiness requires authoritative metadata', () pendingMeta: true, }); expect(result.patches).not.toContainEqual(expect.objectContaining({ synced: true })); + expect(result.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: true, + }); }); it('bypasses synthetic done and heals poisoned ready flags when metaSynced is false', async () => { @@ -659,7 +286,6 @@ describe('context graph subscribe readiness requires authoritative metadata', () expect(result.statusResponse).toMatchObject({ jobId: result.response.catchup.jobId, status: 'done', - attemptStatus: 'done', convergence: { state: 'complete', verified: { @@ -873,147 +499,6 @@ describe('context graph subscribe readiness requires authoritative metadata', () }); }); - it('coalesces a running VM-only selection and serially upgrades it to VM plus SWM', async () => { - const result = await subscribe({ - hasConfirmedMeta: true, - includeSharedMemory: false, - result: privateDataOnlyResult(), - coalescedUpgradeResult: publicDurableAndSharedMemoryResult(), - initial: { - subscribed: false, - synced: false, - sharedMemorySynced: false, - metaSynced: true, - }, - }); - - expect(result.coalescedResponse).toMatchObject({ - subscribed: result.response.subscribed, - catchup: { - status: 'queued', - includeWorkspace: true, - }, - }); - expect(result.coalescedResponse.catchup.jobId).not.toBe( - result.response.catchup.jobId, - ); - expect(result.runRequests).toEqual([ - { - contextGraphId: result.response.subscribed, - includeSharedMemory: false, - }, - { - contextGraphId: result.response.subscribed, - includeSharedMemory: true, - }, - ]); - expect(result.job).toMatchObject({ - jobId: result.response.catchup.jobId, - includeWorkspace: false, - status: 'done', - }); - expect(result.coalescedJob).toMatchObject({ - jobId: result.coalescedResponse.catchup.jobId, - includeWorkspace: true, - status: 'done', - }); - expect(result.statusResponse.convergence).toMatchObject({ - state: 'complete', - required: { sharedMemory: false }, - missing: [], - }); - expect(result.coalescedStatusResponse.convergence).toMatchObject({ - state: 'complete', - required: { sharedMemory: true }, - missing: [], - }); - }); - - it('keeps VM-only success stable when a coalesced SWM upgrade fails', async () => { - const result = await subscribe({ - hasConfirmedMeta: true, - includeSharedMemory: false, - result: privateDataOnlyResult(), - coalescedUpgradeResult: privateDataOnlyResult(), - initial: { - subscribed: false, - synced: false, - sharedMemorySynced: false, - metaSynced: true, - }, - }); - - expect(result.runRequests).toEqual([ - { - contextGraphId: result.response.subscribed, - includeSharedMemory: false, - }, - { - contextGraphId: result.response.subscribed, - includeSharedMemory: true, - }, - ]); - expect(result.job).toMatchObject({ - includeWorkspace: false, - status: 'done', - }); - expect(result.statusResponse).toMatchObject({ - status: 'done', - includeSharedMemory: false, - convergence: { - state: 'complete', - required: { sharedMemory: false }, - missing: [], - }, - }); - expect(result.coalescedJob).toMatchObject({ - includeWorkspace: true, - status: 'unreachable', - error: expect.stringContaining('requested data plane'), - }); - expect(result.coalescedStatusResponse).toMatchObject({ - status: 'unreachable', - includeSharedMemory: true, - convergence: { - state: 'partial', - required: { sharedMemory: true }, - missing: ['sharedMemory'], - }, - }); - }); - - it('reports live completion when automatic retry recovers a failed foreground attempt', async () => { - const result = await subscribe({ - hasConfirmedMeta: true, - isPrivate: true, - result: privateSharedMemoryOnlyResult(), - simulateAutomaticRecovery: true, - initial: { - subscribed: false, - synced: false, - sharedMemorySynced: false, - metaSynced: true, - }, - }); - - expect(result.job.status).toBe('unreachable'); - expect(result.statusResponse).toMatchObject({ - status: 'done', - attemptStatus: 'unreachable', - attemptError: expect.stringContaining('durable VM'), - completedAfterAttempt: true, - convergence: { - state: 'complete', - verified: { - metadata: true, - durable: true, - sharedMemory: true, - }, - missing: [], - }, - }); - }); - it('does not promote positive durable inserts when the plane also timed out', async () => { const partial = privateDataOnlyResult(); if (!partial.diagnostics?.durable) throw new Error('durable diagnostics missing'); diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index 2a721bcd7..a0cc0e47d 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -994,7 +994,11 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { it('marks timeout-after-response catchup as failed rather than unreachable', async () => { const contextGraphId = 'catchup-timeout-response-' + Math.random().toString(36).slice(2, 8); - const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map() }; + const catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + inFlightByContextGraph: new Map(), + }; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { run: async () => ({ @@ -1144,7 +1148,11 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { it('marks local scheduler deferral retryable without setting shared-memory completion', async () => { const contextGraphId = 'catchup-local-deferral-' + Math.random().toString(36).slice(2, 8); - const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map() }; + const catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + inFlightByContextGraph: new Map(), + }; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { run: async () => ({ @@ -2403,6 +2411,7 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), + inFlightByContextGraph: new Map(), }; const previousCatchupRunner = daemonState.catchupRunner; // Benign runner: the queued job runs fire-and-forget after the response and diff --git a/packages/cli/test/helpers/context-graph-catchup-fixtures.ts b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts new file mode 100644 index 000000000..c9530b62b --- /dev/null +++ b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts @@ -0,0 +1,119 @@ +import type { CatchupJobResult } from '../../src/catchup-runner.js'; + +export function cleanEmptyResult(): CatchupJobResult { + return { + connectedPeers: 1, + totalPeers: 1, + selectedPeers: 1, + syncCapablePeers: 1, + peersTried: 1, + peersResponded: 1, + peersSucceeded: 1, + dataSynced: 0, + sharedMemorySynced: 0, + denied: false, + deniedPeers: 0, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 0, emptyPeers: 1 }, + sharedMemory: { verifiedDataPeers: 0, emptyPeers: 1 }, + }, + diagnostics: { + noProtocolPeers: 0, + durable: { + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 0, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 1, + metaOnlyResponses: 0, + dataRejectedMissingMeta: 0, + rejectedKcs: 0, + failedPeers: 0, + failedPhases: 0, + }, + sharedMemory: { + fetchedMetaTriples: 0, + fetchedDataTriples: 0, + insertedMetaTriples: 0, + insertedDataTriples: 0, + bytesReceived: 0, + resumedPhases: 0, + timedOutPhases: 0, + completedPhases: 2, + checkpointAdvances: 0, + emptyResponses: 1, + droppedDataTriples: 0, + failedPeers: 0, + failedPhases: 0, + }, + }, + }; +} + +export function privateMetaOnlyResult(): CatchupJobResult { + const result = cleanEmptyResult(); + if (!result.diagnostics?.durable) throw new Error('durable diagnostics missing'); + result.diagnostics.durable.emptyResponses = 0; + result.diagnostics.durable.fetchedMetaTriples = 7; + result.diagnostics.durable.insertedMetaTriples = 1; + result.diagnostics.durable.metaOnlyResponses = 1; + if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); + result.cleanPlaneCompletions.durable.emptyPeers = 0; + return result; +} + +export function privateDataOnlyResult(): CatchupJobResult { + const result = cleanEmptyResult(); + if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { + throw new Error('catch-up diagnostics missing'); + } + result.dataSynced = 3; + result.diagnostics.durable.emptyResponses = 0; + result.diagnostics.durable.fetchedDataTriples = 3; + result.diagnostics.durable.insertedDataTriples = 3; + result.diagnostics.sharedMemory.emptyResponses = 0; + result.diagnostics.sharedMemory.completedPhases = 0; + result.diagnostics.sharedMemory.timedOutPhases = 1; + if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); + result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; + result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 0, emptyPeers: 0 }; + return result; +} + +export function privateSharedMemoryOnlyResult(): CatchupJobResult { + const result = cleanEmptyResult(); + if (!result.diagnostics?.sharedMemory) { + throw new Error('shared-memory diagnostics missing'); + } + result.sharedMemorySynced = 4; + result.diagnostics.sharedMemory.emptyResponses = 0; + result.diagnostics.sharedMemory.fetchedDataTriples = 4; + result.diagnostics.sharedMemory.insertedDataTriples = 4; + if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); + result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; + return result; +} + +export function publicDurableAndSharedMemoryResult(): CatchupJobResult { + const result = cleanEmptyResult(); + if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { + throw new Error('catch-up diagnostics missing'); + } + if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); + result.dataSynced = 3; + result.sharedMemorySynced = 4; + result.diagnostics.durable.emptyResponses = 0; + result.diagnostics.durable.fetchedDataTriples = 3; + result.diagnostics.durable.insertedDataTriples = 3; + result.diagnostics.sharedMemory.emptyResponses = 0; + result.diagnostics.sharedMemory.fetchedDataTriples = 4; + result.diagnostics.sharedMemory.insertedDataTriples = 4; + result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; + result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; + return result; +} diff --git a/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts new file mode 100644 index 000000000..3446cbfcc --- /dev/null +++ b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts @@ -0,0 +1,337 @@ +import { createServer, type Server } from 'node:http'; +import type { + CatchupJobResult, + CatchupRunRequest, +} from '../../src/catchup-runner.js'; +import { handleContextGraphRoutes } from '../../src/daemon/routes/context-graph.js'; +import { handleQueryRoutes } from '../../src/daemon/routes/query.js'; +import { daemonState } from '../../src/daemon/state.js'; +import type { CatchupJob } from '../../src/daemon/types.js'; +import { cleanEmptyResult } from './context-graph-catchup-fixtures.js'; + +type SyncMode = 'on-demand' | 'always-on'; + +export interface SubscribeRouteHarnessOptions { + initial?: Record; + hasConfirmedMeta: boolean; + hasConfirmedMetaAfterCatchup?: boolean; + strictHasConfirmedMeta?: boolean; + locallyCurated?: boolean; + isPrivate?: boolean; + allowedAgents?: string[]; + callerAddress?: string; + readiness?: { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + updatedAt?: number; + }; + runner?: ( + request: CatchupRunRequest, + callNumber: number, + ) => Promise | CatchupJobResult; +} + +export class ContextGraphSubscribeRouteHarness { + readonly contextGraphId = `readiness-${Math.random().toString(36).slice(2, 8)}`; + readonly state = new Map>(); + readonly patches: Array> = []; + readonly runRequests: CatchupRunRequest[] = []; + readonly subscribeCalls: Array<{ + id: string; + options: { syncMode?: SyncMode } | undefined; + }> = []; + readonly metadataCheckOptions: Array< + { rejectUnregisteredPlaceholder?: boolean } | undefined + > = []; + + private readonly previousCatchupRunner = daemonState.catchupRunner; + private readonly catchupTracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + inFlightByContextGraph: new Map(), + }; + private server: Server | undefined; + private addressPort = 0; + private runCallsValue = 0; + private readinessValue: + | { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + updatedAt: number; + } + | undefined; + + private constructor(private readonly options: SubscribeRouteHarnessOptions) { + if (options.initial) { + this.state.set(this.contextGraphId, { ...options.initial }); + } + this.readinessValue = options.readiness + ? { + ...options.readiness, + updatedAt: options.readiness.updatedAt ?? Date.now(), + } + : undefined; + } + + static async create( + options: SubscribeRouteHarnessOptions, + ): Promise { + const harness = new ContextGraphSubscribeRouteHarness(options); + await harness.start(); + return harness; + } + + get runCalls(): number { + return this.runCallsValue; + } + + get readiness(): Record | undefined { + return this.readinessValue; + } + + async postSubscribe(input?: { + includeSharedMemory?: boolean; + syncMode?: unknown; + }): Promise<{ status: number; body: any }> { + const response = await fetch( + `http://127.0.0.1:${this.addressPort}/api/context-graph/subscribe`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + contextGraphId: this.contextGraphId, + includeSharedMemory: input?.includeSharedMemory ?? true, + ...(input?.syncMode !== undefined ? { syncMode: input.syncMode } : {}), + }), + }, + ); + return { status: response.status, body: await response.json() }; + } + + async waitForJob(jobId: string | undefined): Promise { + if (!jobId) return undefined; + for (let i = 0; i < 50; i += 1) { + const job = this.catchupTracker.jobs.get(jobId); + if (job?.finishedAt) return job; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + return this.catchupTracker.jobs.get(jobId); + } + + getJob(jobId: string | undefined): CatchupJob | undefined { + return jobId ? this.catchupTracker.jobs.get(jobId) : undefined; + } + + async getStatus(jobId: string): Promise { + return fetch( + `http://127.0.0.1:${this.addressPort}/api/sync/catchup-status?jobId=${encodeURIComponent(jobId)}`, + ).then((response) => response.json()); + } + + async getStatusByContextGraph(): Promise { + return fetch( + `http://127.0.0.1:${this.addressPort}/api/sync/catchup-status?contextGraphId=${encodeURIComponent(this.contextGraphId)}`, + ).then((response) => response.json()); + } + + setCompleteReadiness(updatedAt = Date.now()): void { + this.readinessValue = { + version: 1, + durableVerified: true, + sharedMemoryVerified: true, + updatedAt, + }; + this.state.set(this.contextGraphId, { + ...this.state.get(this.contextGraphId), + synced: true, + sharedMemorySynced: true, + metaSynced: true, + pendingMeta: false, + }); + } + + async close(): Promise { + daemonState.catchupRunner = this.previousCatchupRunner; + if (!this.server) return; + const server = this.server; + this.server = undefined; + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + + private async start(): Promise { + daemonState.catchupRunner = { + run: async (request) => { + this.runCallsValue += 1; + this.runRequests.push(request); + return this.options.runner?.(request, this.runCallsValue) ?? cleanEmptyResult(); + }, + close: async () => {}, + }; + + const agent = { + getContextGraphAllowedAgents: async () => this.options.allowedAgents ?? [], + getSubscribedContextGraphs: () => this.state, + subscribeToContextGraph: ( + id: string, + options?: { syncMode?: SyncMode }, + ) => { + this.subscribeCalls.push({ id, options }); + const previous = this.state.get(id); + const effectiveSyncMode = previous?.subscribed && previous.syncMode === 'always-on' + ? 'always-on' + : options?.syncMode ?? previous?.syncMode ?? 'always-on'; + const applied = { + ...previous, + subscribed: true, + synced: previous?.synced ?? false, + syncMode: effectiveSyncMode, + }; + this.state.set(id, applied); + return applied; + }, + markContextGraphSubscriptionState: ( + id: string, + patch: Record, + ) => { + this.patches.push({ ...patch }); + this.state.set(id, { ...this.state.get(id), ...patch }); + }, + hasConfirmedMetaState: async ( + _id: string, + options?: { rejectUnregisteredPlaceholder?: boolean }, + ) => { + this.metadataCheckOptions.push(options); + if ( + options?.rejectUnregisteredPlaceholder === true && + this.options.strictHasConfirmedMeta !== undefined + ) { + return this.options.strictHasConfirmedMeta; + } + return this.runCallsValue > 0 + ? this.options.hasConfirmedMetaAfterCatchup ?? this.options.hasConfirmedMeta + : this.options.hasConfirmedMeta; + }, + isCuratorOf: async () => this.options.locallyCurated ?? false, + isPrivateContextGraph: async () => this.options.isPrivate ?? false, + resolveAgentByToken: () => undefined, + getDefaultAgentAddress: () => + this.options.callerAddress ?? '0x0000000000000000000000000000000000000001', + }; + + this.server = createServer(async (request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + const routeContext = { + req: request, + res: response, + agent, + publisherControl: {}, + publisherRuntime: null, + config: { auth: { enabled: false } }, + startedAt: Date.now(), + dashDb: { + getContextGraphReadinessProvenance: () => this.readinessValue ?? null, + setContextGraphReadinessProvenance: ( + _id: string, + next: { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + }, + ) => { + this.readinessValue = { ...next, updatedAt: Date.now() }; + }, + }, + opWallets: {}, + network: {}, + tracker: {}, + memoryManager: {}, + bridgeAuthToken: undefined, + nodeVersion: 'test', + nodeCommit: 'test', + catchupTracker: this.catchupTracker, + extractionRegistry: {}, + fileStore: {}, + extractionStatus: new Map(), + assertionImportLocks: new Map(), + vectorStore: {}, + embeddingProvider: null, + validTokens: new Set(), + apiHost: '127.0.0.1', + apiPortRef: { value: 0 }, + routePlugins: [], + url, + path: url.pathname, + requestToken: undefined, + requestAgentAddress: undefined, + } as any; + await handleContextGraphRoutes(routeContext); + if (!response.writableEnded) await handleQueryRoutes(routeContext); + if (!response.writableEnded) { + response.statusCode = 404; + response.end(); + } + }); + await new Promise((resolve) => + this.server!.listen(0, '127.0.0.1', resolve)); + const address = this.server.address(); + if (!address || typeof address === 'string') { + throw new Error('route test server did not bind'); + } + this.addressPort = address.port; + } +} + +export interface RunSubscribeScenarioOptions + extends Omit { + result?: CatchupJobResult; + includeSharedMemory?: boolean; + syncMode?: unknown; +} + +export async function runSubscribeScenario( + options: RunSubscribeScenarioOptions, +): Promise<{ + response: any; + responseStatus: number; + job: CatchupJob | undefined; + runCalls: number; + runRequests: CatchupRunRequest[]; + subscribeCalls: ContextGraphSubscribeRouteHarness['subscribeCalls']; + state: Record; + patches: Array>; + readiness: Record | undefined; + statusResponse: any; + metadataCheckOptions: ContextGraphSubscribeRouteHarness['metadataCheckOptions']; +}> { + const harness = await ContextGraphSubscribeRouteHarness.create({ + ...options, + runner: () => options.result ?? cleanEmptyResult(), + }); + try { + const posted = await harness.postSubscribe({ + includeSharedMemory: options.includeSharedMemory, + syncMode: options.syncMode, + }); + const jobId = posted.body.catchup?.jobId as string | undefined; + const job = await harness.waitForJob(jobId); + return { + response: posted.body, + responseStatus: posted.status, + job, + runCalls: harness.runCalls, + runRequests: [...harness.runRequests], + subscribeCalls: [...harness.subscribeCalls], + state: harness.state.get(harness.contextGraphId) ?? {}, + patches: [...harness.patches], + readiness: harness.readiness, + statusResponse: jobId ? await harness.getStatus(jobId) : null, + metadataCheckOptions: [...harness.metadataCheckOptions], + }; + } finally { + await harness.close(); + } +} diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index 04464f9d5..79a6ded33 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -47,7 +47,12 @@ export default defineConfig({ // Private-CG bootstrap readiness: clean-empty responses are only // terminal when authoritative metadata has been confirmed. 'test/context-graph-subscribe-readiness.test.ts', + 'test/context-graph-catchup-coalescing-route.test.ts', + 'test/context-graph-catchup-coordinator.test.ts', 'test/context-graph-catchup-readiness.test.ts', + 'test/catchup-status-response.test.ts', + 'test/catchup-status-convergence-route.test.ts', + 'test/catchup-status-cli.test.ts', 'test/context-graph-readiness-migration.test.ts', // R9 — PCA advisory wire derivation (pure) + CLI register-agent output // rendering (in-process, mocked ApiClient). No hardhat/daemon. From 81a74df9fa1ad80264ab115aba33dff8b9c31858 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:16:56 +0200 Subject: [PATCH 14/23] fix(sync): model catch-up scopes and refresh latest views --- .../context-graph-catchup-coordinator.ts | 362 +++++++++++------- packages/cli/src/daemon/types.ts | 33 +- ...ext-graph-catchup-coalescing-route.test.ts | 81 ++++ .../context-graph-catchup-coordinator.test.ts | 75 +++- 4 files changed, 398 insertions(+), 153 deletions(-) diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts index 42beb67d6..e8ea6fffb 100644 --- a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -4,10 +4,13 @@ import type { CatchupJobResult } from '../catchup-result-wire.js'; import { catchupResultHasCleanResponse, classifyContextGraphCatchupReadiness, + type ContextGraphCatchupReadinessClassification, } from '../context-graph-readiness.js'; import type { CatchupCoordinator, + CatchupExecution, CatchupJob, + CatchupScope, CatchupTracker, } from './types.js'; @@ -42,6 +45,16 @@ export interface ContextGraphCatchupCoordinatorEffects { trace?: (message: string) => void; } +type CatchupResultClassification = + | ContextGraphCatchupReadinessClassification + | { + jobStatus: 'deferred'; + error: string; + readinessPatch?: undefined; + statePatch?: undefined; + eventPayload?: undefined; + }; + export class ContextGraphCatchupCoordinatorService { private readonly now: () => number; private readonly createJobId: () => string; @@ -64,37 +77,47 @@ export class ContextGraphCatchupCoordinatorService { }): CatchupJob | undefined { const coordinator = this.inFlightByContextGraph.get(input.contextGraphId); if (!coordinator) return undefined; - const baseJob = this.tracker.jobs.get(coordinator.baseJobId); - const upgradeJob = coordinator.upgradeJobId - ? this.tracker.jobs.get(coordinator.upgradeJobId) - : undefined; - const narrowProjectionJob = coordinator.narrowProjectionJobId - ? this.tracker.jobs.get(coordinator.narrowProjectionJobId) - : undefined; - const active = [baseJob, upgradeJob].some((candidate) => - candidate?.status === 'queued' || candidate?.status === 'running'); - if (!baseJob || !active) return undefined; - - if (!input.includeSharedMemory) { - if (!baseJob.includeWorkspace) return baseJob; - if (narrowProjectionJob) return narrowProjectionJob; + const hasActiveExecution = coordinator.executions.some((execution) => { + const job = this.tracker.jobs.get(execution.jobId); + return job?.status === 'queued' || job?.status === 'running'; + }); + if (!hasActiveExecution) return undefined; - const createdProjection = this.createJob(input.contextGraphId, false); - coordinator.narrowProjectionJobId = createdProjection.jobId; - this.tracker.latestByContextGraph.set( - input.contextGraphId, - createdProjection.jobId, - ); - return createdProjection; + const requestedScope = this.toScope(input.includeSharedMemory); + const existingView = coordinator.viewsByScope.get(requestedScope); + if (existingView) { + const existingJob = this.tracker.jobs.get(existingView.jobId); + if (existingJob) return this.markLatest(existingJob); } - if (baseJob.includeWorkspace) return baseJob; - if (upgradeJob) return upgradeJob; + if (requestedScope === 'durable') { + const broadExecution = coordinator.executions.find((execution) => + execution.scope === 'durable-and-shared-memory' && + this.isExecutionActive(execution)); + if (!broadExecution) return undefined; - const createdUpgrade = this.createJob(input.contextGraphId, true); - coordinator.upgradeJobId = createdUpgrade.jobId; - this.tracker.latestByContextGraph.set(input.contextGraphId, createdUpgrade.jobId); - return createdUpgrade; + const projection = this.createJob(input.contextGraphId, requestedScope); + coordinator.viewsByScope.set(requestedScope, { + jobId: projection.jobId, + scope: requestedScope, + sourceExecutionJobId: broadExecution.jobId, + kind: 'projection', + }); + return this.markLatest(projection); + } + + const upgrade = this.createJob(input.contextGraphId, requestedScope); + const execution: CatchupExecution = { + jobId: upgrade.jobId, + scope: requestedScope, + }; + coordinator.executions.push(execution); + coordinator.viewsByScope.set(requestedScope, { + jobId: upgrade.jobId, + scope: requestedScope, + kind: 'execution', + }); + return this.markLatest(upgrade); } /** Start one detached serialized worker for a fresh per-CG catch-up. */ @@ -103,23 +126,35 @@ export class ContextGraphCatchupCoordinatorService { includeSharedMemory: boolean; readinessBeforeCatchup: ContextGraphReadinessProvenance; }): CatchupJob { - const job = this.createJob(input.contextGraphId, input.includeSharedMemory); - this.tracker.latestByContextGraph.set(input.contextGraphId, job.jobId); + const scope = this.toScope(input.includeSharedMemory); + const job = this.createJob(input.contextGraphId, scope); + const execution: CatchupExecution = { jobId: job.jobId, scope }; const coordinator: CatchupCoordinator = { contextGraphId: input.contextGraphId, - baseJobId: job.jobId, + executions: [execution], + viewsByScope: new Map([ + [scope, { + jobId: job.jobId, + scope, + kind: 'execution', + }], + ]), }; this.inFlightByContextGraph.set(input.contextGraphId, coordinator); this.pruneCompletedJobs(); - void this.run(coordinator, job, input.readinessBeforeCatchup); - return job; + void this.run(coordinator, input.readinessBeforeCatchup); + return this.markLatest(job); + } + + private toScope(includeSharedMemory: boolean): CatchupScope { + return includeSharedMemory ? 'durable-and-shared-memory' : 'durable'; } - private createJob(contextGraphId: string, includeSharedMemory: boolean): CatchupJob { + private createJob(contextGraphId: string, scope: CatchupScope): CatchupJob { const job: CatchupJob = { jobId: this.createJobId(), contextGraphId, - includeWorkspace: includeSharedMemory, + includeWorkspace: scope === 'durable-and-shared-memory', status: 'queued', queuedAt: this.now(), }; @@ -127,13 +162,21 @@ export class ContextGraphCatchupCoordinatorService { return job; } + private markLatest(job: CatchupJob): CatchupJob { + this.tracker.latestByContextGraph.set(job.contextGraphId, job.jobId); + return job; + } + + private isExecutionActive(execution: CatchupExecution): boolean { + const job = this.tracker.jobs.get(execution.jobId); + return job?.status === 'queued' || job?.status === 'running'; + } + private pruneCompletedJobs(): void { const activeJobIds = new Set(); for (const coordinator of this.inFlightByContextGraph.values()) { - activeJobIds.add(coordinator.baseJobId); - if (coordinator.upgradeJobId) activeJobIds.add(coordinator.upgradeJobId); - if (coordinator.narrowProjectionJobId) { - activeJobIds.add(coordinator.narrowProjectionJobId); + for (const view of coordinator.viewsByScope.values()) { + activeJobIds.add(view.jobId); } } while (this.tracker.jobs.size > 100) { @@ -154,51 +197,42 @@ export class ContextGraphCatchupCoordinatorService { private async run( coordinator: CatchupCoordinator, - baseJob: CatchupJob, readinessBeforeCatchup: ContextGraphReadinessProvenance, ): Promise { - let attemptJob = baseJob; - try { - let readinessBeforeAttempt = readinessBeforeCatchup; - while (true) { - await this.runAttempt(attemptJob, readinessBeforeAttempt); - if (attemptJob === baseJob && baseJob.includeWorkspace) { - await this.settleNarrowProjection( + for (let index = 0; index < coordinator.executions.length; index += 1) { + const execution = coordinator.executions[index]; + const job = this.tracker.jobs.get(execution.jobId); + if (!job || job.status !== 'queued') continue; + const readinessBeforeAttempt = index === 0 + ? readinessBeforeCatchup + : this.effects.readReadiness(coordinator.contextGraphId); + + try { + const attempt = await this.runAttempt(job, readinessBeforeAttempt); + await this.settleProjectionViews( coordinator, - baseJob, + execution, + attempt.result, readinessBeforeAttempt, ); - } - if ( - coordinator.upgradeJobId && - !attemptJob.includeWorkspace && - attemptJob.status !== 'denied' - ) { - const upgradeJob = this.tracker.jobs.get(coordinator.upgradeJobId); - if (!upgradeJob) break; - attemptJob = upgradeJob; - readinessBeforeAttempt = this.effects.readReadiness( - coordinator.contextGraphId, + if (attempt.status === 'denied') { + this.settleRemainingExecutionsFrom(coordinator, index + 1, job); + break; + } + } catch (error) { + job.error = error instanceof Error ? error.message : String(error); + job.status = 'failed'; + job.finishedAt = this.now(); + this.settleProjectionViewsFrom(coordinator, execution, job); + this.settleRemainingExecutionsFrom(coordinator, index + 1, job); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${coordinator.contextGraphId} threw: ${job.error}`, ); - continue; - } - if (!attemptJob.includeWorkspace) { - this.settleQueuedJobFrom(coordinator.upgradeJobId, attemptJob); + break; } - break; } - } catch (error) { - attemptJob.error = error instanceof Error ? error.message : String(error); - attemptJob.status = 'failed'; - attemptJob.finishedAt = this.now(); - this.settleQueuedJobFrom(coordinator.upgradeJobId, attemptJob); - this.settleQueuedJobFrom(coordinator.narrowProjectionJobId, attemptJob); - this.effects.trace?.( - `[catchup] job=${attemptJob.jobId} contextGraph=${coordinator.contextGraphId} threw: ${attemptJob.error}`, - ); } finally { - attemptJob.finishedAt ??= this.now(); if (this.inFlightByContextGraph.get(coordinator.contextGraphId) === coordinator) { this.inFlightByContextGraph.delete(coordinator.contextGraphId); } @@ -208,7 +242,7 @@ export class ContextGraphCatchupCoordinatorService { private async runAttempt( job: CatchupJob, readinessBeforeCatchup: ContextGraphReadinessProvenance, - ): Promise { + ): Promise<{ result: CatchupJobResult; status: CatchupJob['status'] }> { job.status = 'running'; job.startedAt ??= this.now(); this.effects.trace?.( @@ -218,58 +252,70 @@ export class ContextGraphCatchupCoordinatorService { contextGraphId: job.contextGraphId, includeSharedMemory: job.includeWorkspace, }); - await this.applyResult(job, result, readinessBeforeCatchup, true); + const classification = await this.classifyResult( + job, + result, + readinessBeforeCatchup, + ); + this.applyExecutionEffects(job.contextGraphId, classification); + this.settleClassifiedJob(job, result, classification); + return { result, status: job.status }; } - private async applyResult( + private async classifyResult( job: CatchupJob, result: CatchupJobResult, readinessBeforeCatchup: ContextGraphReadinessProvenance, - applyEffects: boolean, - ): Promise { - job.result = result; - job.error = undefined; - + ): Promise { if (result.deferredBackpressure > 0 && !result.denied) { - job.status = 'deferred'; - job.error = 'Sync deferred by local scheduler backpressure; retry when capacity is available.'; - } else { - const inspectReadiness = catchupResultHasCleanResponse(result); - const hasConfirmedMeta = inspectReadiness - ? await this.effects.hasConfirmedMeta(job.contextGraphId) - : false; - const isPrivate = hasConfirmedMeta - ? await this.effects.isPrivate(job.contextGraphId) - : false; - const classification = classifyContextGraphCatchupReadiness({ - result, - includeSharedMemory: job.includeWorkspace, - hasConfirmedMeta, - isPrivate, - readinessBeforeCatchup, - }); - job.status = classification.jobStatus; - job.error = classification.error; - if (applyEffects && classification.readinessPatch) { - this.effects.writeReadiness(job.contextGraphId, classification.readinessPatch); - } - if (applyEffects && classification.statePatch) { - this.effects.markSubscriptionState( - job.contextGraphId, - classification.statePatch, - ); - } - if (applyEffects && classification.eventPayload) { - this.effects.emitProjectSynced( - job.contextGraphId, - classification.eventPayload, - ); - } - if (job.status === 'done' && result.deferredBackpressure > 0) { - job.status = 'deferred'; - job.error = 'Sync deferred by local scheduler backpressure; retry when capacity is available.'; - } + return { + jobStatus: 'deferred' as const, + error: 'Sync deferred by local scheduler backpressure; retry when capacity is available.', + readinessPatch: undefined, + statePatch: undefined, + eventPayload: undefined, + }; } + + const inspectReadiness = catchupResultHasCleanResponse(result); + const hasConfirmedMeta = inspectReadiness + ? await this.effects.hasConfirmedMeta(job.contextGraphId) + : false; + const isPrivate = hasConfirmedMeta + ? await this.effects.isPrivate(job.contextGraphId) + : false; + return classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory: job.includeWorkspace, + hasConfirmedMeta, + isPrivate, + readinessBeforeCatchup, + }); + } + + private applyExecutionEffects( + contextGraphId: string, + classification: CatchupResultClassification, + ): void { + if (classification.readinessPatch) { + this.effects.writeReadiness(contextGraphId, classification.readinessPatch); + } + if (classification.statePatch) { + this.effects.markSubscriptionState(contextGraphId, classification.statePatch); + } + if (classification.eventPayload) { + this.effects.emitProjectSynced(contextGraphId, classification.eventPayload); + } + } + + private settleClassifiedJob( + job: CatchupJob, + result: CatchupJobResult, + classification: CatchupResultClassification, + ): void { + job.result = result; + job.status = classification.jobStatus; + job.error = classification.error; job.finishedAt = this.now(); this.effects.trace?.( `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} status=${job.status} ` + @@ -279,11 +325,8 @@ export class ContextGraphCatchupCoordinatorService { ); } - private settleQueuedJobFrom( - targetJobId: string | undefined, - source: CatchupJob, - ): void { - const target = targetJobId ? this.tracker.jobs.get(targetJobId) : undefined; + private settleQueuedJobFrom(targetJobId: string, source: CatchupJob): void { + const target = this.tracker.jobs.get(targetJobId); if (!target || target.status !== 'queued') return; target.status = source.status; target.error = source.error; @@ -292,34 +335,61 @@ export class ContextGraphCatchupCoordinatorService { target.finishedAt = this.now(); } - private async settleNarrowProjection( + private async settleProjectionViews( coordinator: CatchupCoordinator, - source: CatchupJob, + execution: CatchupExecution, + result: CatchupJobResult, readinessBeforeCatchup: ContextGraphReadinessProvenance, ): Promise { - const projection = coordinator.narrowProjectionJobId - ? this.tracker.jobs.get(coordinator.narrowProjectionJobId) - : undefined; - if (!projection || projection.status !== 'queued') return; + const source = this.tracker.jobs.get(execution.jobId); + if (!source) return; + for (const view of coordinator.viewsByScope.values()) { + if ( + view.kind !== 'projection' || + view.sourceExecutionJobId !== execution.jobId + ) continue; + const projection = this.tracker.jobs.get(view.jobId); + if (!projection || projection.status !== 'queued') continue; + projection.status = 'running'; + projection.startedAt = source.startedAt; + try { + const classification = await this.classifyResult( + projection, + result, + readinessBeforeCatchup, + ); + this.settleClassifiedJob(projection, result, classification); + } catch (error) { + projection.status = 'failed'; + projection.error = error instanceof Error ? error.message : String(error); + projection.finishedAt = this.now(); + } + } + } - if (!source.result || source.status === 'failed') { - this.settleQueuedJobFrom(coordinator.narrowProjectionJobId, source); - return; + private settleProjectionViewsFrom( + coordinator: CatchupCoordinator, + execution: CatchupExecution, + source: CatchupJob, + ): void { + for (const view of coordinator.viewsByScope.values()) { + if ( + view.kind === 'projection' && + view.sourceExecutionJobId === execution.jobId + ) { + this.settleQueuedJobFrom(view.jobId, source); + } } + } - projection.status = 'running'; - projection.startedAt = source.startedAt; - try { - await this.applyResult( - projection, - source.result, - readinessBeforeCatchup, - false, - ); - } catch (error) { - projection.status = 'failed'; - projection.error = error instanceof Error ? error.message : String(error); - projection.finishedAt = this.now(); + private settleRemainingExecutionsFrom( + coordinator: CatchupCoordinator, + startIndex: number, + source: CatchupJob, + ): void { + for (const execution of coordinator.executions.slice(startIndex)) { + this.settleQueuedJobFrom(execution.jobId, source); + this.settleProjectionViewsFrom(coordinator, execution, source); } } } diff --git a/packages/cli/src/daemon/types.ts b/packages/cli/src/daemon/types.ts index 90babcd4e..3ed1f1493 100644 --- a/packages/cli/src/daemon/types.ts +++ b/packages/cli/src/daemon/types.ts @@ -19,16 +19,37 @@ export interface CatchupJob { error?: string; } +export type CatchupScope = 'durable' | 'durable-and-shared-memory'; + +export interface CatchupExecution { + jobId: string; + scope: CatchupScope; +} + +export type CatchupJobView = + | { + jobId: string; + scope: CatchupScope; + kind: 'execution'; + } + | { + jobId: string; + scope: CatchupScope; + sourceExecutionJobId: string; + kind: 'projection'; + }; + /** - * Mutable orchestration state for one serialized per-CG catch-up. Public job - * records stay immutable in scope; a later SWM upgrade receives its own jobId - * while reusing this coordinator and the same background worker. + * Mutable orchestration state for one serialized per-CG catch-up. Executions + * describe actual runner work; views describe the immutable public job for + * each requested scope. A narrow view can project a broad execution without + * pretending to be another execution, while a wider request queues one real + * serialized execution. */ export interface CatchupCoordinator { contextGraphId: string; - baseJobId: string; - upgradeJobId?: string; - narrowProjectionJobId?: string; + executions: CatchupExecution[]; + viewsByScope: Map; } export interface CatchupTracker { diff --git a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts index 90ba289cf..8bb42a8f3 100644 --- a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts +++ b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts @@ -17,6 +17,87 @@ function deferred(): { } describe('context graph catch-up route coalescing', () => { + it('makes a reused broad job latest after broad, narrow, broad requests', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async () => { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return publicDurableAndSharedMemoryResult(); + }, + }); + + try { + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await firstRunStarted.promise; + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: narrow.body.catchup.jobId, + includeSharedMemory: false, + }); + + const reusedBroad = await harness.postSubscribe({ includeSharedMemory: true }); + expect(reusedBroad.body.catchup.jobId).toBe(broad.body.catchup.jobId); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: broad.body.catchup.jobId, + includeSharedMemory: true, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + + it('makes a reused narrow job latest after narrow, upgrade, narrow requests', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async (_request, callNumber) => { + if (callNumber === 1) { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return privateDataOnlyResult(); + } + return publicDurableAndSharedMemoryResult(); + }, + }); + + try { + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + await firstRunStarted.promise; + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: broad.body.catchup.jobId, + includeSharedMemory: true, + }); + + const reusedNarrow = await harness.postSubscribe({ includeSharedMemory: false }); + expect(reusedNarrow.body.catchup.jobId).toBe(narrow.body.catchup.jobId); + await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ + jobId: narrow.body.catchup.jobId, + includeSharedMemory: false, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + it('serially upgrades an active VM-only request with a distinct VM plus SWM job', async () => { const firstRunStarted = deferred(); const releaseFirstRun = deferred(); diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index cd3ade503..ed9a8f5a5 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -41,6 +41,7 @@ function deniedResult() { function coordinatorFixture(options: { failUpgrade?: boolean; baseOutcome?: 'success' | 'throw' | 'denied'; + blockBroadBase?: boolean; } = {}) { const tracker: CatchupTracker = { jobs: new Map(), @@ -56,8 +57,10 @@ function coordinatorFixture(options: { let sequence = 0; const firstRunStarted = deferred(); const releaseFirstRun = deferred(); + let runNumber = 0; const run = vi.fn(async (request: { includeSharedMemory: boolean }) => { - if (!request.includeSharedMemory) { + runNumber += 1; + if (runNumber === 1 && (!request.includeSharedMemory || options.blockBroadBase)) { firstRunStarted.resolve(); await releaseFirstRun.promise; if (options.baseOutcome === 'throw') { @@ -92,6 +95,76 @@ function coordinatorFixture(options: { } describe('ContextGraphCatchupCoordinatorService', () => { + it('refreshes latest status when broad, narrow, then broad reuses existing views', async () => { + const fixture = coordinatorFixture({ blockBroadBase: true }); + const broad = fixture.service.start({ + contextGraphId: 'cg:broad-narrow-broad', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + + const narrow = fixture.service.coalesceActive({ + contextGraphId: 'cg:broad-narrow-broad', + includeSharedMemory: false, + }); + expect(fixture.tracker.latestByContextGraph.get('cg:broad-narrow-broad')) + .toBe(narrow?.jobId); + + const reusedBroad = fixture.service.coalesceActive({ + contextGraphId: 'cg:broad-narrow-broad', + includeSharedMemory: true, + }); + expect(reusedBroad?.jobId).toBe(broad.jobId); + expect(fixture.tracker.latestByContextGraph.get('cg:broad-narrow-broad')) + .toBe(broad.jobId); + + fixture.releaseFirstRun.resolve(); + await waitForJob(broad); + if (!narrow) throw new Error('narrow view missing'); + await waitForJob(narrow); + }); + + it('refreshes latest status when narrow, upgrade, then narrow reuses the base view', async () => { + const fixture = coordinatorFixture(); + const narrow = fixture.service.start({ + contextGraphId: 'cg:narrow-broad-narrow', + includeSharedMemory: false, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + + const broad = fixture.service.coalesceActive({ + contextGraphId: 'cg:narrow-broad-narrow', + includeSharedMemory: true, + }); + expect(fixture.tracker.latestByContextGraph.get('cg:narrow-broad-narrow')) + .toBe(broad?.jobId); + + const reusedNarrow = fixture.service.coalesceActive({ + contextGraphId: 'cg:narrow-broad-narrow', + includeSharedMemory: false, + }); + expect(reusedNarrow?.jobId).toBe(narrow.jobId); + expect(fixture.tracker.latestByContextGraph.get('cg:narrow-broad-narrow')) + .toBe(narrow.jobId); + + fixture.releaseFirstRun.resolve(); + await waitForJob(narrow); + if (!broad) throw new Error('broad upgrade missing'); + await waitForJob(broad); + }); + it('keeps job scopes immutable and serializes one wider upgrade', async () => { const fixture = coordinatorFixture(); const base = fixture.service.start({ From cdd6b05e5476cac379c57cafe2edfad7d7226665 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:31:18 +0200 Subject: [PATCH 15/23] refactor(sync): narrow catch-up route dependencies --- packages/cli/src/context-graph-readiness.ts | 11 ++++- .../context-graph-catchup-route-adapter.ts | 49 +++++++++++++++++++ .../cli/src/daemon/routes/context-graph.ts | 38 +++++--------- 3 files changed, 70 insertions(+), 28 deletions(-) create mode 100644 packages/cli/src/daemon/context-graph-catchup-route-adapter.ts diff --git a/packages/cli/src/context-graph-readiness.ts b/packages/cli/src/context-graph-readiness.ts index be35f223c..bcf914ee3 100644 --- a/packages/cli/src/context-graph-readiness.ts +++ b/packages/cli/src/context-graph-readiness.ts @@ -50,6 +50,15 @@ export interface ContextGraphReadinessPatch { sharedMemoryVerified: boolean; } +/** Narrow capability required to decide whether local CG metadata is trusted. */ +export interface ContextGraphMetadataAuthority { + isCuratorOf?: (contextGraphId: string) => Promise; + hasConfirmedMetaState: ( + contextGraphId: string, + options?: { rejectUnregisteredPlaceholder?: boolean }, + ) => Promise; +} + export type ContextGraphReadinessPlanes = Omit< ContextGraphConvergenceSnapshot, 'observedAt' @@ -628,7 +637,7 @@ async function withContextGraphReadinessMutationLock( * must reject the legacy unregistered placeholder shape. */ export async function hasAuthoritativeContextGraphMetadata(input: { - agent: DKGAgent; + agent: ContextGraphMetadataAuthority; contextGraphId: string; }): Promise { const locallyCurated = typeof input.agent.isCuratorOf === 'function' diff --git a/packages/cli/src/daemon/context-graph-catchup-route-adapter.ts b/packages/cli/src/daemon/context-graph-catchup-route-adapter.ts new file mode 100644 index 000000000..2a57430ed --- /dev/null +++ b/packages/cli/src/daemon/context-graph-catchup-route-adapter.ts @@ -0,0 +1,49 @@ +import type { DKGAgent } from '@origintrail-official/dkg-agent'; +import { DKGEvent } from '@origintrail-official/dkg-core'; +import type { DashboardDB } from '@origintrail-official/dkg-node-ui'; +import type { CatchupRunner } from '../catchup-runner.js'; +import { + hasAuthoritativeContextGraphMetadata, + readContextGraphReadiness, + writeContextGraphReadiness, +} from '../context-graph-readiness.js'; +import { ContextGraphCatchupCoordinatorService } from './context-graph-catchup-coordinator.js'; +import type { CatchupTracker } from './types.js'; + +/** + * Bind daemon-owned persistence, agent, and event effects once at the route + * boundary. The subscribe route only chooses a scope and delegates + * start/coalescing; readiness classification and side effects stay behind the + * coordinator's narrow API. + */ +export function createContextGraphCatchupRouteAdapter(input: { + tracker: CatchupTracker; + runner: Pick; + readinessStore: DashboardDB; + agent: DKGAgent; + trace?: (message: string) => void; +}): ContextGraphCatchupCoordinatorService { + return new ContextGraphCatchupCoordinatorService(input.tracker, { + runner: input.runner, + readReadiness: (contextGraphId) => + readContextGraphReadiness(input.readinessStore, contextGraphId), + hasConfirmedMeta: (contextGraphId) => + hasAuthoritativeContextGraphMetadata({ + agent: input.agent, + contextGraphId, + }), + isPrivate: (contextGraphId) => + input.agent.isPrivateContextGraph(contextGraphId).catch(() => true), + writeReadiness: (contextGraphId, patch) => + writeContextGraphReadiness(input.readinessStore, contextGraphId, patch), + markSubscriptionState: (contextGraphId, patch) => + input.agent.markContextGraphSubscriptionState(contextGraphId, patch), + emitProjectSynced: (contextGraphId, payload) => { + input.agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { + contextGraphId, + ...payload, + }); + }, + ...(input.trace ? { trace: input.trace } : {}), + }); +} diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index fba019ae8..a5099382e 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -65,7 +65,7 @@ import { VmReconcileQueueFullError, VmReconcileUnavailableError, } from '@origintrail-official/dkg-agent'; -import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri, SYSTEM_CONTEXT_GRAPHS } from '@origintrail-official/dkg-core'; +import { computeNetworkId, createOperationContext, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri, SYSTEM_CONTEXT_GRAPHS } from '@origintrail-official/dkg-core'; import { findReservedSubjectPrefix, isSkolemizedUri } from '@origintrail-official/dkg-publisher'; import { DashboardDB, @@ -115,7 +115,6 @@ import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type P import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; import { classifyExistingContextGraphReadiness, - hasAuthoritativeContextGraphMetadata, readContextGraphReadiness, writeContextGraphReadiness, } from '../../context-graph-readiness.js'; @@ -168,7 +167,7 @@ import { type CatchupJob, type CatchupTracker, } from '../types.js'; -import { ContextGraphCatchupCoordinatorService } from '../context-graph-catchup-coordinator.js'; +import { createContextGraphCatchupRouteAdapter } from '../context-graph-catchup-route-adapter.js'; import { type MarkItDownTarget, manifestRepoRoot, @@ -1740,30 +1739,15 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise readContextGraphReadiness(dashDb, id), - hasConfirmedMeta: (id) => hasAuthoritativeContextGraphMetadata({ - agent, - contextGraphId: id, - }), - isPrivate: (id) => agent.isPrivateContextGraph(id).catch(() => true), - writeReadiness: (id, patch) => writeContextGraphReadiness(dashDb, id, patch), - markSubscriptionState: (id, patch) => - agent.markContextGraphSubscriptionState(id, patch), - emitProjectSynced: (id, payload) => { - agent.eventBus?.emit?.(DKGEvent.PROJECT_SYNCED, { - contextGraphId: id, - ...payload, - }); - }, - ...(DEBUG_SYNC_TRACE - ? { trace: (message: string) => console.log(message) } - : {}), - }, - ); + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: dashDb, + agent, + ...(DEBUG_SYNC_TRACE + ? { trace: (message: string) => console.log(message) } + : {}), + }); const existingJobId = catchupTracker.latestByContextGraph.get(contextGraphId); const existingJob = existingJobId ? catchupTracker.jobs.get(existingJobId) : undefined; let readinessBeforeCatchup = readContextGraphReadiness(dashDb, contextGraphId); From f9570f8ed5337288e1a8af6da7a6997136b1a2b4 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:38:14 +0200 Subject: [PATCH 16/23] fix(sync): preserve legacy tracker initialization --- .../context-graph-catchup-coordinator.ts | 5 ++++- .../context-graph-catchup-coordinator.test.ts | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts index e8ea6fffb..b3b619ec7 100644 --- a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -67,7 +67,10 @@ export class ContextGraphCatchupCoordinatorService { this.now = effects.now ?? Date.now; this.createJobId = effects.createJobId ?? (() => `${this.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`); - this.inFlightByContextGraph = tracker.inFlightByContextGraph; + // Preserve the historical two-map tracker shape used by embedded callers + // and older route fixtures. The coordinator owns the new orchestration + // index, so it can provision that index without changing those call sites. + this.inFlightByContextGraph = tracker.inFlightByContextGraph ??= new Map(); } /** Reuse active work while preserving each caller's immutable plane scope. */ diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index ed9a8f5a5..612cb3b29 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -95,6 +95,28 @@ function coordinatorFixture(options: { } describe('ContextGraphCatchupCoordinatorService', () => { + it('provisions orchestration state for the historical two-map tracker shape', () => { + const tracker = { + jobs: new Map(), + latestByContextGraph: new Map(), + } as CatchupTracker; + const service = new ContextGraphCatchupCoordinatorService(tracker, { + runner: { run: vi.fn() }, + readReadiness: vi.fn(), + hasConfirmedMeta: vi.fn(), + isPrivate: vi.fn(), + writeReadiness: vi.fn(), + markSubscriptionState: vi.fn(), + emitProjectSynced: vi.fn(), + }); + + expect(service.coalesceActive({ + contextGraphId: 'cg:legacy-tracker', + includeSharedMemory: true, + })).toBeUndefined(); + expect(tracker.inFlightByContextGraph).toBeInstanceOf(Map); + }); + it('refreshes latest status when broad, narrow, then broad reuses existing views', async () => { const fixture = coordinatorFixture({ blockBroadBase: true }); const broad = fixture.service.start({ From 9871c758eb93ca7f4a9e600db99c436b0efa33b2 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:41:07 +0200 Subject: [PATCH 17/23] fix(sync): centralize authoritative metadata checks --- packages/cli/src/daemon/routes/context-graph.ts | 3 ++- .../cli/test/context-graph-subscribe-readiness.test.ts | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index a5099382e..9e96b02bd 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -115,6 +115,7 @@ import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type P import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; import { classifyExistingContextGraphReadiness, + hasAuthoritativeContextGraphMetadata, readContextGraphReadiness, writeContextGraphReadiness, } from '../../context-graph-readiness.js'; @@ -1774,7 +1775,7 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise false); + await hasAuthoritativeContextGraphMetadata({ agent, contextGraphId }); const existingReadiness = classifyExistingContextGraphReadiness({ subscription: existingSub, readiness: readinessBeforeCatchup, diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index f5c88d209..2c2b72c82 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -124,7 +124,10 @@ describe('context graph subscribe readiness requires authoritative metadata', () it('revalidates a stale metaSynced=true bit before returning synthetic done', async () => { const result = await subscribe({ - hasConfirmedMeta: false, + // The permissive legacy check sees a placeholder, but the canonical + // readiness boundary must reject it for a remotely curated graph. + hasConfirmedMeta: true, + strictHasConfirmedMeta: false, initial: { subscribed: true, synced: true, @@ -143,6 +146,9 @@ describe('context graph subscribe readiness requires authoritative metadata', () metaSynced: false, pendingMeta: true, }); + expect(result.metadataCheckOptions).toContainEqual({ + rejectUnregisteredPlaceholder: true, + }); }); it('does not restore stale true provenance after metadata arrives during an unclean catch-up', async () => { From a3a45e3736f93d9c80ba13e672cca3f57de92577 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 13:52:21 +0200 Subject: [PATCH 18/23] fix(sync): preserve incomplete-round invariants --- packages/cli/src/catchup-result-wire.ts | 16 ++++++- packages/cli/src/catchup-runner.ts | 43 ++++-------------- .../context-graph-catchup-coordinator.ts | 15 ++++++- .../context-graph-catchup-coordinator.test.ts | 45 +++++++++++++++++-- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/packages/cli/src/catchup-result-wire.ts b/packages/cli/src/catchup-result-wire.ts index 460252fc0..470e173af 100644 --- a/packages/cli/src/catchup-result-wire.ts +++ b/packages/cli/src/catchup-result-wire.ts @@ -1,9 +1,23 @@ -/** Per-plane clean-completion evidence safe to expose on the status wire. */ +/** + * Per-plane clean-completion evidence accumulated across the peers this run + * contacted and safe to expose on the status wire. + */ export interface CatchupPlaneCompletionEvidence { verifiedDataPeers: number; + /** Peers that cleanly verified one or more V2 KAs with no public triples. */ verifiedPrivateOnlyPeers?: number; emptyPeers: number; + /** + * The metadata-resolved curator cleanly completed this plane while hosting + * the graph and carrying no data at all. + */ authorityEmptyPeers?: number; + /** + * Peers that answered this plane but whose round did not complete cleanly. + * This prevents one clean-empty response plus one incomplete-empty response + * from being misread as unanimous empty. Pure transport failures remain in + * the plane diagnostics rather than this counter. + */ incompleteResponders?: number; } diff --git a/packages/cli/src/catchup-runner.ts b/packages/cli/src/catchup-runner.ts index cba5164fd..6d2727669 100644 --- a/packages/cli/src/catchup-runner.ts +++ b/packages/cli/src/catchup-runner.ts @@ -13,9 +13,15 @@ import { type SyncPeerResolution, } from '@origintrail-official/dkg-agent'; import { PROTOCOL_SYNC } from '@origintrail-official/dkg-core'; -import type { CatchupJobResult } from './catchup-result-wire.js'; +import type { + CatchupJobResult, + CatchupPlaneCompletionEvidence, +} from './catchup-result-wire.js'; -export type { CatchupJobResult } from './catchup-result-wire.js'; +export type { + CatchupJobResult, + CatchupPlaneCompletionEvidence, +} from './catchup-result-wire.js'; const SYNC_PROTOCOL_CHECK_ATTEMPTS = 3; const SYNC_PROTOCOL_CHECK_DELAY_MS = 500; @@ -423,39 +429,6 @@ export function catchupPlaneCompletedWithoutFailure( return classifyDurableProgress(progress, { complete }).completedWithoutFailure; } -/** Per-plane clean-completion evidence accumulated across the peers this run contacted. */ -export interface CatchupPlaneCompletionEvidence { - verifiedDataPeers: number; - /** Peers that cleanly verified one or more V2 KAs with no public triples. */ - verifiedPrivateOnlyPeers?: number; - emptyPeers: number; - /** - * The metadata-resolved curator cleanly completed this plane while hosting - * the graph and carrying no data at all. See - * {@link catchupPlaneProvenByUnanimousEmpty}. - */ - authorityEmptyPeers?: number; - /** - * Peers that ANSWERED this plane but whose round did not complete cleanly. - * - * Every other field here records what a peer proved. This one records what a - * peer left unresolved, and it exists because the absence of a peer from the - * positive counters is ambiguous: `catchupPeerPlaneEvidence` returns an - * all-zero record for an incomplete round, so a peer that answered EMPTY but - * did not finish paging is indistinguishable from a peer that was never - * contacted. That ambiguity is invisible to the round diagnostics too — an - * explicit `complete: false` is not a transport failure, so it never reaches - * `failedPeers`. - * - * Without it, a round of one clean-empty peer plus one incomplete-empty peer - * reads as unanimously empty. Pure transport failures are deliberately NOT - * counted here: an unreachable stranger is already `failedPeers`, and folding - * it in would pin legitimately empty graphs in a retry loop on a lossy - * network. - */ - incompleteResponders?: number; -} - /** The aggregate per-plane counters a whole-round verdict is allowed to consult. */ export interface CatchupPlaneRoundDiagnostics { fetchedMetaTriples?: number; diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts index b3b619ec7..1831afa18 100644 --- a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -287,13 +287,26 @@ export class ContextGraphCatchupCoordinatorService { const isPrivate = hasConfirmedMeta ? await this.effects.isPrivate(job.contextGraphId) : false; - return classifyContextGraphCatchupReadiness({ + const classification = classifyContextGraphCatchupReadiness({ result, includeSharedMemory: job.includeWorkspace, hasConfirmedMeta, isPrivate, readinessBeforeCatchup, }); + // Denial can coexist with usable data from another peer. If local + // admission also deferred part of that mixed round, the clean data must + // not turn the attempt into success: finalizeCatchup deliberately leaves + // any backpressured round incomplete. Preserve a pure ACL denial, but + // downgrade an otherwise-successful mixed result before effects are + // applied so no readiness bit is frozen from partial work. + if (result.deferredBackpressure > 0 && classification.jobStatus === 'done') { + return { + jobStatus: 'deferred', + error: 'Sync deferred by local scheduler backpressure; retry when capacity is available.', + }; + } + return classification; } private applyExecutionEffects( diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index 612cb3b29..a3d409f33 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -1,5 +1,6 @@ import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg-node-ui'; import { describe, expect, it, vi } from 'vitest'; +import type { CatchupJobResult } from '../src/catchup-runner.js'; import { ContextGraphCatchupCoordinatorService } from '../src/daemon/context-graph-catchup-coordinator.js'; import type { CatchupJob, CatchupTracker } from '../src/daemon/types.js'; import { @@ -42,6 +43,7 @@ function coordinatorFixture(options: { failUpgrade?: boolean; baseOutcome?: 'success' | 'throw' | 'denied'; blockBroadBase?: boolean; + result?: CatchupJobResult; } = {}) { const tracker: CatchupTracker = { jobs: new Map(), @@ -60,6 +62,7 @@ function coordinatorFixture(options: { let runNumber = 0; const run = vi.fn(async (request: { includeSharedMemory: boolean }) => { runNumber += 1; + if (options.result) return options.result; if (runNumber === 1 && (!request.includeSharedMemory || options.blockBroadBase)) { firstRunStarted.resolve(); await releaseFirstRun.promise; @@ -73,15 +76,20 @@ function coordinatorFixture(options: { ? privateDataOnlyResult() : publicDurableAndSharedMemoryResult(); }); + const writeReadiness = vi.fn((_contextGraphId: string, patch: { + durableVerified: boolean; + sharedMemoryVerified: boolean; + }) => { + readiness = { ...readiness, ...patch, updatedAt: readiness.updatedAt + 1 }; + }); + const markSubscriptionState = vi.fn(); const service = new ContextGraphCatchupCoordinatorService(tracker, { runner: { run }, readReadiness: () => readiness, hasConfirmedMeta: async () => true, isPrivate: async () => false, - writeReadiness: (_contextGraphId, patch) => { - readiness = { ...readiness, ...patch, updatedAt: readiness.updatedAt + 1 }; - }, - markSubscriptionState: vi.fn(), + writeReadiness, + markSubscriptionState, emitProjectSynced: vi.fn(), createJobId: () => `job-${++sequence}`, }); @@ -91,10 +99,39 @@ function coordinatorFixture(options: { run, firstRunStarted, releaseFirstRun, + writeReadiness, + markSubscriptionState, }; } describe('ContextGraphCatchupCoordinatorService', () => { + it('keeps denied mixed progress deferred when local backpressure left the round incomplete', async () => { + const result = publicDurableAndSharedMemoryResult(); + result.denied = true; + result.deniedPeers = 1; + result.deferredBackpressure = 1; + const fixture = coordinatorFixture({ result }); + const job = fixture.service.start({ + contextGraphId: 'cg:denied-deferred', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + + await waitForJob(job); + + expect(job).toMatchObject({ + status: 'deferred', + error: expect.stringContaining('local scheduler backpressure'), + }); + expect(fixture.writeReadiness).not.toHaveBeenCalled(); + expect(fixture.markSubscriptionState).not.toHaveBeenCalled(); + }); + it('provisions orchestration state for the historical two-map tracker shape', () => { const tracker = { jobs: new Map(), From fd9ac439111347157d03b8bf6fe33a774bd7b591 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 14:01:22 +0200 Subject: [PATCH 19/23] refactor(sync): make tracker compatibility explicit --- .../daemon/context-graph-catchup-coordinator.ts | 16 ++++++++++++---- packages/cli/src/daemon/types.ts | 7 ++++++- .../context-graph-catchup-coordinator.test.ts | 8 +++++--- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts index 1831afa18..c336fb1b1 100644 --- a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -55,6 +55,17 @@ type CatchupResultClassification = eventPayload?: undefined; }; +/** Normalize the explicitly optional legacy tracker boundary exactly once. */ +export function getOrCreateCatchupCoordinatorIndex( + tracker: CatchupTracker, +): Map { + const existing = tracker.inFlightByContextGraph; + if (existing) return existing; + const created = new Map(); + tracker.inFlightByContextGraph = created; + return created; +} + export class ContextGraphCatchupCoordinatorService { private readonly now: () => number; private readonly createJobId: () => string; @@ -67,10 +78,7 @@ export class ContextGraphCatchupCoordinatorService { this.now = effects.now ?? Date.now; this.createJobId = effects.createJobId ?? (() => `${this.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`); - // Preserve the historical two-map tracker shape used by embedded callers - // and older route fixtures. The coordinator owns the new orchestration - // index, so it can provision that index without changing those call sites. - this.inFlightByContextGraph = tracker.inFlightByContextGraph ??= new Map(); + this.inFlightByContextGraph = getOrCreateCatchupCoordinatorIndex(tracker); } /** Reuse active work while preserving each caller's immutable plane scope. */ diff --git a/packages/cli/src/daemon/types.ts b/packages/cli/src/daemon/types.ts index 3ed1f1493..cde8556fe 100644 --- a/packages/cli/src/daemon/types.ts +++ b/packages/cli/src/daemon/types.ts @@ -55,5 +55,10 @@ export interface CatchupCoordinator { export interface CatchupTracker { jobs: Map; latestByContextGraph: Map; - inFlightByContextGraph: Map; + /** + * Coordinator-only index added after the original two-map tracker contract. + * Optional at the public boundary so embedded callers can supply that legacy + * shape; the coordinator normalizes it through one canonical helper. + */ + inFlightByContextGraph?: Map; } diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index a3d409f33..5b1c558a7 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -133,10 +133,12 @@ describe('ContextGraphCatchupCoordinatorService', () => { }); it('provisions orchestration state for the historical two-map tracker shape', () => { - const tracker = { + // Compile-time regression: the public tracker boundary explicitly accepts + // the historical two-map shape without a cast. + const tracker: CatchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - } as CatchupTracker; + }; const service = new ContextGraphCatchupCoordinatorService(tracker, { runner: { run: vi.fn() }, readReadiness: vi.fn(), @@ -263,7 +265,7 @@ describe('ContextGraphCatchupCoordinatorService', () => { ]); expect(base).toMatchObject({ includeWorkspace: false, status: 'done' }); expect(upgrade).toMatchObject({ includeWorkspace: true, status: 'done' }); - expect(fixture.tracker.inFlightByContextGraph.has('cg:one')).toBe(false); + expect(fixture.tracker.inFlightByContextGraph?.has('cg:one')).toBe(false); }); it('does not retroactively fail VM-only success when the wider upgrade is incomplete', async () => { From 693ae68e1899be9e39489e34c912a31eccb1e139 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 14:50:17 +0200 Subject: [PATCH 20/23] fix(sync): preserve successful catch-up attempts --- .../cli/src/daemon/catchup-status-response.ts | 50 ++++- packages/cli/src/daemon/routes/query.ts | 25 +-- .../catchup-status-convergence-route.test.ts | 40 +++- .../cli/test/catchup-status-response.test.ts | 18 ++ .../context-graph-subscribe-readiness.test.ts | 10 + .../helpers/context-graph-catchup-fixtures.ts | 208 +++++++++++++----- .../context-graph-subscribe-route-harness.ts | 12 + 7 files changed, 286 insertions(+), 77 deletions(-) diff --git a/packages/cli/src/daemon/catchup-status-response.ts b/packages/cli/src/daemon/catchup-status-response.ts index 88d4f0e41..8cce83bed 100644 --- a/packages/cli/src/daemon/catchup-status-response.ts +++ b/packages/cli/src/daemon/catchup-status-response.ts @@ -2,6 +2,13 @@ import type { CatchupConvergenceStatus, CatchupStatusResponse, } from '../catchup-status-wire.js'; +import { + describeContextGraphConvergence, + hasAuthoritativeContextGraphMetadata, + readContextGraphReadiness, + type ContextGraphMetadataAuthority, + type ContextGraphReadinessStore, +} from '../context-graph-readiness.js'; import type { CatchupJob } from './types.js'; export type { @@ -23,7 +30,10 @@ export function toCatchupStatusResponse( job.status === 'unreachable'); const invalidatedAfterAttempt = job.status === 'done' && convergence !== undefined && - convergence.state !== 'complete'; + convergence.state !== 'complete' && + convergence.readinessUpdatedAt !== undefined && + job.finishedAt !== undefined && + convergence.readinessUpdatedAt > job.finishedAt; const status = completedAfterAttempt ? 'done' : invalidatedAfterAttempt @@ -48,3 +58,41 @@ export function toCatchupStatusResponse( ...(completedAfterAttempt ? { completedAfterAttempt: true } : {}), }; } + +export interface CatchupStatusAgent extends ContextGraphMetadataAuthority { + getSubscribedContextGraphs(): ReadonlyMap; +} + +/** Load live convergence and apply the canonical status projection together. */ +export async function loadCatchupStatusResponse(input: { + job: CatchupJob; + agent: CatchupStatusAgent; + readinessStore: Partial; + observedAt?: number; +}): Promise { + const subscription = input.agent.getSubscribedContextGraphs() + .get(input.job.contextGraphId); + const hasConfirmedMeta = await hasAuthoritativeContextGraphMetadata({ + agent: input.agent, + contextGraphId: input.job.contextGraphId, + }); + const convergence: CatchupConvergenceStatus = { + ...describeContextGraphConvergence({ + readiness: readContextGraphReadiness( + input.readinessStore, + input.job.contextGraphId, + ), + includeSharedMemory: input.job.includeWorkspace, + hasConfirmedMeta, + ...(input.observedAt === undefined + ? {} + : { observedAt: input.observedAt }), + }), + syncMode: subscription?.syncMode ?? 'always-on', + automaticRetryActive: subscription?.subscribed === true, + }; + return toCatchupStatusResponse(input.job, convergence); +} diff --git a/packages/cli/src/daemon/routes/query.ts b/packages/cli/src/daemon/routes/query.ts index 12579e97a..64d26b237 100644 --- a/packages/cli/src/daemon/routes/query.ts +++ b/packages/cli/src/daemon/routes/query.ts @@ -118,11 +118,6 @@ export { export type { ApiQueryPriority } from '../api-query-priority.js'; import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; -import { - describeContextGraphConvergence, - hasAuthoritativeContextGraphMetadata, - readContextGraphReadiness, -} from '../../context-graph-readiness.js'; import { loadTokens, httpAuthGuard, extractBearerToken } from '../../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; import { MarkItDownConverter, isMarkItDownAvailable, extractFromMarkdown, extractWithLlm } from '../../extraction/index.js'; @@ -172,7 +167,7 @@ import { type CatchupJob, type CatchupTracker, } from '../types.js'; -import { toCatchupStatusResponse } from '../catchup-status-response.js'; +import { loadCatchupStatusResponse } from '../catchup-status-response.js'; import { type MarkItDownTarget, manifestRepoRoot, @@ -1038,22 +1033,12 @@ export async function handleQueryRoutes(ctx: RequestContext): Promise { }); } - const subscription = agent.getSubscribedContextGraphs().get(job.contextGraphId); - const hasConfirmedMeta = await hasAuthoritativeContextGraphMetadata({ + const response = await loadCatchupStatusResponse({ + job, agent, - contextGraphId: job.contextGraphId, + readinessStore: dashDb, }); - const convergence = { - ...describeContextGraphConvergence({ - readiness: readContextGraphReadiness(dashDb, job.contextGraphId), - includeSharedMemory: job.includeWorkspace, - hasConfirmedMeta, - }), - syncMode: subscription?.syncMode ?? 'always-on', - automaticRetryActive: subscription?.subscribed === true, - }; - - return jsonResponse(res, 200, toCatchupStatusResponse(job, convergence)); + return jsonResponse(res, 200, response); } // POST /api/verify diff --git a/packages/cli/test/catchup-status-convergence-route.test.ts b/packages/cli/test/catchup-status-convergence-route.test.ts index 8f698c67c..75951beb1 100644 --- a/packages/cli/test/catchup-status-convergence-route.test.ts +++ b/packages/cli/test/catchup-status-convergence-route.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { privateSharedMemoryOnlyResult } from './helpers/context-graph-catchup-fixtures.js'; +import { + lossyPublicEmptyResult, + privateSharedMemoryOnlyResult, +} from './helpers/context-graph-catchup-fixtures.js'; import { ContextGraphSubscribeRouteHarness } from './helpers/context-graph-subscribe-route-harness.js'; async function runIncompletePrivateAttempt( @@ -25,6 +28,41 @@ async function runIncompletePrivateAttempt( } describe('catch-up status live convergence', () => { + it('preserves a lossy public empty-round success without persisted completion', async () => { + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + isPrivate: false, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: () => lossyPublicEmptyResult(), + }); + try { + const response = await harness.postSubscribe({ includeSharedMemory: false }); + const jobId = response.body.catchup.jobId as string; + const job = await harness.waitForJob(jobId); + expect(job).toMatchObject({ status: 'done' }); + + await expect(harness.getStatus(jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'partial', + verified: { + metadata: true, + durable: false, + sharedMemory: false, + }, + missing: ['durable'], + }, + }); + } finally { + await harness.close(); + } + }); + it('keeps a failed attempt failed when complete readiness predates the job', async () => { const harness = await ContextGraphSubscribeRouteHarness.create({ hasConfirmedMeta: true, diff --git a/packages/cli/test/catchup-status-response.test.ts b/packages/cli/test/catchup-status-response.test.ts index 7c008846e..47ab9faf9 100644 --- a/packages/cli/test/catchup-status-response.test.ts +++ b/packages/cli/test/catchup-status-response.test.ts @@ -88,4 +88,22 @@ describe('catch-up status response', () => { convergence: invalidatedConvergence, }); }); + + it('preserves a successful attempt when incomplete convergence is not a newer invalidation', () => { + const nonPersistedConvergence = { + ...completeConvergence, + state: 'partial' as const, + verified: { metadata: true, durable: false, sharedMemory: false }, + missing: ['durable', 'sharedMemory'] as const, + readinessUpdatedAt: undefined, + }; + + expect(toCatchupStatusResponse(job('done'), nonPersistedConvergence)) + .toMatchObject({ + status: 'done', + convergence: nonPersistedConvergence, + }); + expect(toCatchupStatusResponse(job('done'), nonPersistedConvergence)) + .not.toHaveProperty('attempt'); + }); }); diff --git a/packages/cli/test/context-graph-subscribe-readiness.test.ts b/packages/cli/test/context-graph-subscribe-readiness.test.ts index 2c2b72c82..5c6df707b 100644 --- a/packages/cli/test/context-graph-subscribe-readiness.test.ts +++ b/packages/cli/test/context-graph-subscribe-readiness.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { DKGEvent } from '@origintrail-official/dkg-core'; import { cleanEmptyResult, privateDataOnlyResult, @@ -319,6 +320,15 @@ describe('context graph subscribe readiness requires authoritative metadata', () durableVerified: true, sharedMemoryVerified: true, }); + expect(result.emittedEvents).toContainEqual({ + event: DKGEvent.PROJECT_SYNCED, + payload: { + contextGraphId: result.response.subscribed, + dataSynced: 3, + sharedMemorySynced: 4, + verifiedPrivateOnlyResponses: 0, + }, + }); }); // Issue #2006: an empty response cannot distinguish "hosts an empty graph" diff --git a/packages/cli/test/helpers/context-graph-catchup-fixtures.ts b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts index c9530b62b..a18e79394 100644 --- a/packages/cli/test/helpers/context-graph-catchup-fixtures.ts +++ b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts @@ -1,7 +1,30 @@ import type { CatchupJobResult } from '../../src/catchup-runner.js'; -export function cleanEmptyResult(): CatchupJobResult { - return { +type CleanPlaneCompletions = NonNullable< + CatchupJobResult['cleanPlaneCompletions'] +>; +type CatchupDiagnostics = NonNullable; + +export type CatchupJobResultOverrides = Omit< + Partial, + 'cleanPlaneCompletions' | 'diagnostics' +> & { + cleanPlaneCompletions?: { + durable?: Partial; + sharedMemory?: Partial; + }; + diagnostics?: { + noProtocolPeers?: number; + durable?: Partial; + sharedMemory?: Partial; + }; +}; + +/** Complete canonical result factory; scenario helpers override only their signal. */ +export function makeCatchupJobResult( + overrides: CatchupJobResultOverrides = {}, +): CatchupJobResult { + const defaults = { connectedPeers: 1, totalPeers: 1, selectedPeers: 1, @@ -9,13 +32,21 @@ export function cleanEmptyResult(): CatchupJobResult { peersTried: 1, peersResponded: 1, peersSucceeded: 1, + deferredBackpressure: 0, dataSynced: 0, sharedMemorySynced: 0, denied: false, deniedPeers: 0, cleanPlaneCompletions: { - durable: { verifiedDataPeers: 0, emptyPeers: 1 }, - sharedMemory: { verifiedDataPeers: 0, emptyPeers: 1 }, + durable: { + verifiedDataPeers: 0, + verifiedPrivateOnlyPeers: 0, + emptyPeers: 1, + }, + sharedMemory: { + verifiedDataPeers: 0, + emptyPeers: 1, + }, }, diagnostics: { noProtocolPeers: 0, @@ -31,10 +62,12 @@ export function cleanEmptyResult(): CatchupJobResult { checkpointAdvances: 0, emptyResponses: 1, metaOnlyResponses: 0, + verifiedPrivateOnlyResponses: 0, dataRejectedMissingMeta: 0, rejectedKcs: 0, failedPeers: 0, failedPhases: 0, + deferredBackpressure: 0, }, sharedMemory: { fetchedMetaTriples: 0, @@ -50,70 +83,135 @@ export function cleanEmptyResult(): CatchupJobResult { droppedDataTriples: 0, failedPeers: 0, failedPhases: 0, + deferredBackpressure: 0, + }, + }, + } satisfies CatchupJobResult; + + return { + ...defaults, + ...overrides, + cleanPlaneCompletions: { + durable: { + ...defaults.cleanPlaneCompletions.durable, + ...overrides.cleanPlaneCompletions?.durable, + }, + sharedMemory: { + ...defaults.cleanPlaneCompletions.sharedMemory, + ...overrides.cleanPlaneCompletions?.sharedMemory, + }, + }, + diagnostics: { + noProtocolPeers: + overrides.diagnostics?.noProtocolPeers ?? defaults.diagnostics.noProtocolPeers, + durable: { + ...defaults.diagnostics.durable, + ...overrides.diagnostics?.durable, + }, + sharedMemory: { + ...defaults.diagnostics.sharedMemory, + ...overrides.diagnostics?.sharedMemory, }, }, }; } +export function cleanEmptyResult(): CatchupJobResult { + return makeCatchupJobResult(); +} + export function privateMetaOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable) throw new Error('durable diagnostics missing'); - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedMetaTriples = 7; - result.diagnostics.durable.insertedMetaTriples = 1; - result.diagnostics.durable.metaOnlyResponses = 1; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.durable.emptyPeers = 0; - return result; + return makeCatchupJobResult({ + cleanPlaneCompletions: { + durable: { emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedMetaTriples: 7, + insertedMetaTriples: 1, + metaOnlyResponses: 1, + }, + }, + }); } export function privateDataOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { - throw new Error('catch-up diagnostics missing'); - } - result.dataSynced = 3; - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedDataTriples = 3; - result.diagnostics.durable.insertedDataTriples = 3; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.completedPhases = 0; - result.diagnostics.sharedMemory.timedOutPhases = 1; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 0, emptyPeers: 0 }; - return result; + return makeCatchupJobResult({ + dataSynced: 3, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 1, emptyPeers: 0 }, + sharedMemory: { verifiedDataPeers: 0, emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedDataTriples: 3, + insertedDataTriples: 3, + }, + sharedMemory: { + emptyResponses: 0, + completedPhases: 0, + timedOutPhases: 1, + }, + }, + }); } export function privateSharedMemoryOnlyResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.sharedMemory) { - throw new Error('shared-memory diagnostics missing'); - } - result.sharedMemorySynced = 4; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.fetchedDataTriples = 4; - result.diagnostics.sharedMemory.insertedDataTriples = 4; - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; - return result; + return makeCatchupJobResult({ + sharedMemorySynced: 4, + cleanPlaneCompletions: { + sharedMemory: { verifiedDataPeers: 1, emptyPeers: 0 }, + }, + diagnostics: { + sharedMemory: { + emptyResponses: 0, + fetchedDataTriples: 4, + insertedDataTriples: 4, + }, + }, + }); } export function publicDurableAndSharedMemoryResult(): CatchupJobResult { - const result = cleanEmptyResult(); - if (!result.diagnostics?.durable || !result.diagnostics.sharedMemory) { - throw new Error('catch-up diagnostics missing'); - } - if (!result.cleanPlaneCompletions) throw new Error('clean completion proof missing'); - result.dataSynced = 3; - result.sharedMemorySynced = 4; - result.diagnostics.durable.emptyResponses = 0; - result.diagnostics.durable.fetchedDataTriples = 3; - result.diagnostics.durable.insertedDataTriples = 3; - result.diagnostics.sharedMemory.emptyResponses = 0; - result.diagnostics.sharedMemory.fetchedDataTriples = 4; - result.diagnostics.sharedMemory.insertedDataTriples = 4; - result.cleanPlaneCompletions.durable = { verifiedDataPeers: 1, emptyPeers: 0 }; - result.cleanPlaneCompletions.sharedMemory = { verifiedDataPeers: 1, emptyPeers: 0 }; - return result; + return makeCatchupJobResult({ + dataSynced: 3, + sharedMemorySynced: 4, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 1, emptyPeers: 0 }, + sharedMemory: { verifiedDataPeers: 1, emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedDataTriples: 3, + insertedDataTriples: 3, + }, + sharedMemory: { + emptyResponses: 0, + fetchedDataTriples: 4, + insertedDataTriples: 4, + }, + }, + }); +} + +/** One public host proves empty while another selected peer transport-fails. */ +export function lossyPublicEmptyResult(): CatchupJobResult { + return makeCatchupJobResult({ + connectedPeers: 2, + totalPeers: 2, + selectedPeers: 2, + syncCapablePeers: 2, + peersTried: 2, + peersResponded: 1, + peersSucceeded: 1, + cleanPlaneCompletions: { + durable: { emptyPeers: 1 }, + }, + diagnostics: { + durable: { failedPeers: 1 }, + }, + }); } diff --git a/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts index 3446cbfcc..2ec75858f 100644 --- a/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts +++ b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts @@ -1,4 +1,5 @@ import { createServer, type Server } from 'node:http'; +import { DKGEvent } from '@origintrail-official/dkg-core'; import type { CatchupJobResult, CatchupRunRequest, @@ -44,6 +45,10 @@ export class ContextGraphSubscribeRouteHarness { readonly metadataCheckOptions: Array< { rejectUnregisteredPlaceholder?: boolean } | undefined > = []; + readonly emittedEvents: Array<{ + event: DKGEvent; + payload: unknown; + }> = []; private readonly previousCatchupRunner = daemonState.catchupRunner; private readonly catchupTracker = { @@ -217,6 +222,11 @@ export class ContextGraphSubscribeRouteHarness { }, isCuratorOf: async () => this.options.locallyCurated ?? false, isPrivateContextGraph: async () => this.options.isPrivate ?? false, + eventBus: { + emit: (event: DKGEvent, payload: unknown) => { + this.emittedEvents.push({ event, payload }); + }, + }, resolveAgentByToken: () => undefined, getDefaultAgentAddress: () => this.options.callerAddress ?? '0x0000000000000000000000000000000000000001', @@ -306,6 +316,7 @@ export async function runSubscribeScenario( readiness: Record | undefined; statusResponse: any; metadataCheckOptions: ContextGraphSubscribeRouteHarness['metadataCheckOptions']; + emittedEvents: ContextGraphSubscribeRouteHarness['emittedEvents']; }> { const harness = await ContextGraphSubscribeRouteHarness.create({ ...options, @@ -330,6 +341,7 @@ export async function runSubscribeScenario( readiness: harness.readiness, statusResponse: jobId ? await harness.getStatus(jobId) : null, metadataCheckOptions: [...harness.metadataCheckOptions], + emittedEvents: [...harness.emittedEvents], }; } finally { await harness.close(); From 0fdb3cb67c1ebb0e331f2fe1ea784b0edeafddd9 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 16:10:23 +0200 Subject: [PATCH 21/23] fix(sync): scope coalesced catch-up work --- packages/cli/src/context-graph-readiness.ts | 187 ++++++++-- .../context-graph-catchup-coordinator.ts | 319 +++++++++--------- packages/cli/src/daemon/lifecycle.ts | 13 +- .../cli/src/daemon/routes/context-graph.ts | 13 +- packages/cli/src/daemon/routes/context.ts | 3 + packages/cli/src/daemon/types.ts | 38 +-- ...ext-graph-catchup-coalescing-route.test.ts | 45 +++ .../context-graph-catchup-coordinator.test.ts | 58 +++- .../context-graph-catchup-readiness.test.ts | 42 +++ .../test/daemon-http-behavior-extra.test.ts | 32 +- .../helpers/context-graph-catchup-fixtures.ts | 24 ++ .../context-graph-subscribe-route-harness.ts | 37 +- 12 files changed, 542 insertions(+), 269 deletions(-) diff --git a/packages/cli/src/context-graph-readiness.ts b/packages/cli/src/context-graph-readiness.ts index bcf914ee3..e8ab20425 100644 --- a/packages/cli/src/context-graph-readiness.ts +++ b/packages/cli/src/context-graph-readiness.ts @@ -64,6 +64,17 @@ export type ContextGraphReadinessPlanes = Omit< 'observedAt' >; +interface ResolvedReadinessPlanes { + state: ContextGraphReadinessPlanes['state']; + metadataVerified: boolean; + durableVerified: boolean; + sharedMemoryVerified: boolean; + missingMetadata: boolean; + missingDurable: boolean; + missingRequestedSharedMemory: boolean; + readinessUpdatedAt?: number; +} + function hasCurrentReadinessProvenance( readiness: ContextGraphReadinessProvenance, ): boolean { @@ -76,48 +87,66 @@ function describeResolvedReadinessPlanes(input: { durableEvidence: boolean; sharedMemoryEvidence: boolean; readinessUpdatedAt?: number; -}): ContextGraphReadinessPlanes { +}): ResolvedReadinessPlanes { const metadataVerified = input.hasConfirmedMeta; const durableVerified = metadataVerified && input.durableEvidence; const sharedMemoryVerified = metadataVerified && input.sharedMemoryEvidence; - const missing: ContextGraphConvergencePlane[] = []; - if (!metadataVerified) missing.push('metadata'); - if (!durableVerified) missing.push('durable'); - if (input.includeSharedMemory && !sharedMemoryVerified) missing.push('sharedMemory'); + const missingMetadata = !metadataVerified; + const missingDurable = !durableVerified; + const missingRequestedSharedMemory = + input.includeSharedMemory && !sharedMemoryVerified; const anyVerified = metadataVerified || durableVerified || (input.includeSharedMemory && sharedMemoryVerified); + const complete = !missingMetadata && !missingDurable && + !missingRequestedSharedMemory; + + return { + state: complete ? 'complete' : anyVerified ? 'partial' : 'pending', + metadataVerified, + durableVerified, + sharedMemoryVerified, + missingMetadata, + missingDurable, + missingRequestedSharedMemory, + ...(input.readinessUpdatedAt !== undefined + ? { readinessUpdatedAt: input.readinessUpdatedAt } + : {}), + }; +} +function renderReadinessPlanes( + planes: ResolvedReadinessPlanes, + includeSharedMemory: boolean, +): ContextGraphReadinessPlanes { + const missing: ContextGraphConvergencePlane[] = []; + if (planes.missingMetadata) missing.push('metadata'); + if (planes.missingDurable) missing.push('durable'); + if (planes.missingRequestedSharedMemory) missing.push('sharedMemory'); return { - state: missing.length === 0 ? 'complete' : anyVerified ? 'partial' : 'pending', + state: planes.state, required: { metadata: true, durable: true, - sharedMemory: input.includeSharedMemory, + sharedMemory: includeSharedMemory, }, verified: { - metadata: metadataVerified, - durable: durableVerified, - sharedMemory: sharedMemoryVerified, + metadata: planes.metadataVerified, + durable: planes.durableVerified, + sharedMemory: planes.sharedMemoryVerified, }, missing, - ...(input.readinessUpdatedAt !== undefined - ? { readinessUpdatedAt: input.readinessUpdatedAt } + ...(planes.readinessUpdatedAt !== undefined + ? { readinessUpdatedAt: planes.readinessUpdatedAt } : {}), }; } -/** - * Canonical interpretation of persisted readiness as independently verified - * metadata, durable VM, and optional SWM planes. Every caller that needs to - * decide whether a selected graph is ready must go through this helper so - * metadata loss and readiness-version changes cannot drift between paths. - */ -export function describeReadinessPlanes(input: { +function resolvePersistedReadinessPlanes(input: { readiness: ContextGraphReadinessProvenance; includeSharedMemory: boolean; hasConfirmedMeta: boolean; -}): ContextGraphReadinessPlanes { +}): ResolvedReadinessPlanes { const currentReadinessProvenance = hasCurrentReadinessProvenance(input.readiness); return describeResolvedReadinessPlanes({ hasConfirmedMeta: input.hasConfirmedMeta, @@ -133,16 +162,29 @@ export function describeReadinessPlanes(input: { } /** - * Merge current-run completion evidence with persisted provenance without - * pretending the merged value was itself read from storage. + * Canonical interpretation of persisted readiness as independently verified + * metadata, durable VM, and optional SWM planes. Every caller that needs to + * decide whether a selected graph is ready must go through this helper so + * metadata loss and readiness-version changes cannot drift between paths. */ -export function combineCatchupPlaneEvidence(input: { +export function describeReadinessPlanes(input: { + readiness: ContextGraphReadinessProvenance; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ContextGraphReadinessPlanes { + return renderReadinessPlanes( + resolvePersistedReadinessPlanes(input), + input.includeSharedMemory, + ); +} + +function resolveCatchupPlaneEvidence(input: { readinessBeforeCatchup: ContextGraphReadinessProvenance; durableReadyThisRun: boolean; sharedMemoryReadyThisRun: boolean; includeSharedMemory: boolean; hasConfirmedMeta: boolean; -}): ContextGraphReadinessPlanes { +}): ResolvedReadinessPlanes { const currentReadinessProvenance = hasCurrentReadinessProvenance( input.readinessBeforeCatchup, ); @@ -163,6 +205,23 @@ export function combineCatchupPlaneEvidence(input: { }); } +/** + * Merge current-run completion evidence with persisted provenance without + * pretending the merged value was itself read from storage. + */ +export function combineCatchupPlaneEvidence(input: { + readinessBeforeCatchup: ContextGraphReadinessProvenance; + durableReadyThisRun: boolean; + sharedMemoryReadyThisRun: boolean; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; +}): ContextGraphReadinessPlanes { + return renderReadinessPlanes( + resolveCatchupPlaneEvidence(input), + input.includeSharedMemory, + ); +} + /** * Describe the live, per-plane convergence of one selected context graph. * A VM/SWM proof is only effective while authoritative CG metadata is local; @@ -214,7 +273,7 @@ export function classifyExistingContextGraphReadiness(input: { const currentReadinessProvenance = hasCurrentReadinessProvenance( input.readiness, ); - const planes = describeReadinessPlanes(input); + const planes = resolvePersistedReadinessPlanes(input); const alreadyReady = planes.state === 'complete' && input.subscription.synced === true && @@ -242,8 +301,8 @@ export function classifyExistingContextGraphReadiness(input: { }; } - const durableVerified = planes.verified.durable; - const sharedMemoryVerified = planes.verified.sharedMemory; + const durableVerified = planes.durableVerified; + const sharedMemoryVerified = planes.sharedMemoryVerified; const statePatch = input.subscription.synced !== durableVerified || input.subscription.sharedMemorySynced !== sharedMemoryVerified @@ -400,7 +459,7 @@ function catchupPlaneReadinessThisRun(input: { } export interface ContextGraphCatchupReadinessClassification { - jobStatus: 'done' | 'failed' | 'denied' | 'unreachable'; + jobStatus: 'done' | 'failed' | 'denied' | 'unreachable' | 'deferred'; error?: string; statePatch?: ContextGraphSubscriptionStatePatch; readinessPatch?: ContextGraphReadinessPatch; @@ -411,6 +470,44 @@ export interface ContextGraphCatchupReadinessClassification { }; } +function deferredCatchupClassification(): ContextGraphCatchupReadinessClassification { + return { + jobStatus: 'deferred', + error: 'Sync deferred by local scheduler backpressure; retry when capacity is available.', + }; +} + +/** + * Backpressure is scoped to the planes requested by this view. A durable-only + * projection of a broad execution must not inherit shared-memory deferral. + * Legacy results without plane diagnostics fall back to the aggregate counter + * so rolling-upgrade callers remain fail-closed. + */ +function requestedCatchupBackpressure( + result: CatchupJobResult, + includeSharedMemory: boolean, +): number { + if (includeSharedMemory) return result.deferredBackpressure; + return result.diagnostics?.durable.deferredBackpressure + ?? result.deferredBackpressure; +} + +/** Whether the canonical classifier needs chain-backed CG metadata this round. */ +export function catchupClassificationNeedsMetadata(input: { + result: CatchupJobResult; + includeSharedMemory: boolean; +}): boolean { + const deferredBackpressure = requestedCatchupBackpressure( + input.result, + input.includeSharedMemory, + ); + // A pure local deferral settles before any metadata/readiness branch. Mixed + // denial still needs the full classifier to distinguish denial from usable + // progress, so it must inspect clean evidence normally. + if (deferredBackpressure > 0 && !input.result.denied) return false; + return catchupResultHasCleanResponse(input.result); +} + /** * Canonical policy for converting one catch-up result into externally visible * subscription readiness. The HTTP route gathers live metadata and applies @@ -422,6 +519,30 @@ export function classifyContextGraphCatchupReadiness(input: { hasConfirmedMeta: boolean; isPrivate: boolean; readinessBeforeCatchup: ContextGraphReadinessProvenance; +}): ContextGraphCatchupReadinessClassification { + const deferredBackpressure = requestedCatchupBackpressure( + input.result, + input.includeSharedMemory, + ); + if (deferredBackpressure > 0 && !input.result.denied) { + return deferredCatchupClassification(); + } + + const classification = classifyContextGraphCatchupReadinessWithoutBackpressure(input); + // Pure ACL denial keeps its specific status. When denial coexists with clean + // requested-plane progress, local deferral still prevents that partial round + // from becoming successful or freezing readiness. + return deferredBackpressure > 0 && classification.jobStatus === 'done' + ? deferredCatchupClassification() + : classification; +} + +function classifyContextGraphCatchupReadinessWithoutBackpressure(input: { + result: CatchupJobResult; + includeSharedMemory: boolean; + hasConfirmedMeta: boolean; + isPrivate: boolean; + readinessBeforeCatchup: ContextGraphReadinessProvenance; }): ContextGraphCatchupReadinessClassification { const { result } = input; const durableDataProgress = result.dataSynced > 0; @@ -473,23 +594,23 @@ export function classifyContextGraphCatchupReadiness(input: { currentReadinessProvenance && input.readinessBeforeCatchup.durableVerified; const sharedMemoryVerifiedBefore = currentReadinessProvenance && input.readinessBeforeCatchup.sharedMemoryVerified; - const planes = combineCatchupPlaneEvidence({ + const planes = resolveCatchupPlaneEvidence({ readinessBeforeCatchup: input.readinessBeforeCatchup, durableReadyThisRun, sharedMemoryReadyThisRun, includeSharedMemory: input.includeSharedMemory, hasConfirmedMeta: input.hasConfirmedMeta, }); - const durableVerified = planes.verified.durable; - const sharedMemoryVerified = planes.verified.sharedMemory; + const durableVerified = planes.durableVerified; + const sharedMemoryVerified = planes.sharedMemoryVerified; // What this run is allowed to FREEZE, as opposed to what it reports. These // diverge only for a unanimous-empty verdict, which stays re-derived per run // so that a wrong empty verdict cannot become permanent. const durableVerifiedPersisted = durableVerifiedBefore || durableThisRun.persistable; const sharedMemoryVerifiedPersisted = sharedMemoryVerifiedBefore || sharedMemoryThisRun.persistable; - const missingDurable = planes.missing.includes('durable'); - const missingRequestedSharedMemory = planes.missing.includes('sharedMemory'); + const missingDurable = planes.missingDurable; + const missingRequestedSharedMemory = planes.missingRequestedSharedMemory; const madeIncompleteProgress = (durableDataProgress && !durableReadyThisRun) || (sharedMemoryProgress && !sharedMemoryReadyThisRun); diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts index c336fb1b1..06fa5f404 100644 --- a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -2,13 +2,12 @@ import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg- import type { CatchupRunner } from '../catchup-runner.js'; import type { CatchupJobResult } from '../catchup-result-wire.js'; import { - catchupResultHasCleanResponse, + catchupClassificationNeedsMetadata, classifyContextGraphCatchupReadiness, type ContextGraphCatchupReadinessClassification, } from '../context-graph-readiness.js'; import type { CatchupCoordinator, - CatchupExecution, CatchupJob, CatchupScope, CatchupTracker, @@ -45,31 +44,12 @@ export interface ContextGraphCatchupCoordinatorEffects { trace?: (message: string) => void; } -type CatchupResultClassification = - | ContextGraphCatchupReadinessClassification - | { - jobStatus: 'deferred'; - error: string; - readinessPatch?: undefined; - statePatch?: undefined; - eventPayload?: undefined; - }; - -/** Normalize the explicitly optional legacy tracker boundary exactly once. */ -export function getOrCreateCatchupCoordinatorIndex( - tracker: CatchupTracker, -): Map { - const existing = tracker.inFlightByContextGraph; - if (existing) return existing; - const created = new Map(); - tracker.inFlightByContextGraph = created; - return created; -} +type CatchupResultClassification = ContextGraphCatchupReadinessClassification; export class ContextGraphCatchupCoordinatorService { private readonly now: () => number; private readonly createJobId: () => string; - private readonly inFlightByContextGraph: Map; + private readonly inFlightByContextGraph = new Map(); constructor( private readonly tracker: CatchupTracker, @@ -78,7 +58,6 @@ export class ContextGraphCatchupCoordinatorService { this.now = effects.now ?? Date.now; this.createJobId = effects.createJobId ?? (() => `${this.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`); - this.inFlightByContextGraph = getOrCreateCatchupCoordinatorIndex(tracker); } /** Reuse active work while preserving each caller's immutable plane scope. */ @@ -88,47 +67,26 @@ export class ContextGraphCatchupCoordinatorService { }): CatchupJob | undefined { const coordinator = this.inFlightByContextGraph.get(input.contextGraphId); if (!coordinator) return undefined; - const hasActiveExecution = coordinator.executions.some((execution) => { - const job = this.tracker.jobs.get(execution.jobId); - return job?.status === 'queued' || job?.status === 'running'; - }); - if (!hasActiveExecution) return undefined; + if (!this.hasActiveJob(coordinator)) return undefined; const requestedScope = this.toScope(input.includeSharedMemory); - const existingView = coordinator.viewsByScope.get(requestedScope); - if (existingView) { - const existingJob = this.tracker.jobs.get(existingView.jobId); + const existingJobId = this.jobIdForScope(coordinator, requestedScope); + if (existingJobId) { + const existingJob = this.tracker.jobs.get(existingJobId); if (existingJob) return this.markLatest(existingJob); } if (requestedScope === 'durable') { - const broadExecution = coordinator.executions.find((execution) => - execution.scope === 'durable-and-shared-memory' && - this.isExecutionActive(execution)); - if (!broadExecution) return undefined; - - const projection = this.createJob(input.contextGraphId, requestedScope); - coordinator.viewsByScope.set(requestedScope, { - jobId: projection.jobId, - scope: requestedScope, - sourceExecutionJobId: broadExecution.jobId, - kind: 'projection', - }); - return this.markLatest(projection); + if (coordinator.initialScope !== 'durable-and-shared-memory') return undefined; + const durable = this.createJob(input.contextGraphId, requestedScope); + coordinator.durableJobId = durable.jobId; + return this.markLatest(durable); } - const upgrade = this.createJob(input.contextGraphId, requestedScope); - const execution: CatchupExecution = { - jobId: upgrade.jobId, - scope: requestedScope, - }; - coordinator.executions.push(execution); - coordinator.viewsByScope.set(requestedScope, { - jobId: upgrade.jobId, - scope: requestedScope, - kind: 'execution', - }); - return this.markLatest(upgrade); + if (coordinator.initialScope !== 'durable') return undefined; + const full = this.createJob(input.contextGraphId, requestedScope); + coordinator.fullJobId = full.jobId; + return this.markLatest(full); } /** Start one detached serialized worker for a fresh per-CG catch-up. */ @@ -139,17 +97,12 @@ export class ContextGraphCatchupCoordinatorService { }): CatchupJob { const scope = this.toScope(input.includeSharedMemory); const job = this.createJob(input.contextGraphId, scope); - const execution: CatchupExecution = { jobId: job.jobId, scope }; const coordinator: CatchupCoordinator = { contextGraphId: input.contextGraphId, - executions: [execution], - viewsByScope: new Map([ - [scope, { - jobId: job.jobId, - scope, - kind: 'execution', - }], - ]), + initialScope: scope, + ...(scope === 'durable' + ? { durableJobId: job.jobId } + : { fullJobId: job.jobId }), }; this.inFlightByContextGraph.set(input.contextGraphId, coordinator); this.pruneCompletedJobs(); @@ -178,17 +131,28 @@ export class ContextGraphCatchupCoordinatorService { return job; } - private isExecutionActive(execution: CatchupExecution): boolean { - const job = this.tracker.jobs.get(execution.jobId); - return job?.status === 'queued' || job?.status === 'running'; + private jobIdForScope( + coordinator: CatchupCoordinator, + scope: CatchupScope, + ): string | undefined { + return scope === 'durable' + ? coordinator.durableJobId + : coordinator.fullJobId; + } + + private hasActiveJob(coordinator: CatchupCoordinator): boolean { + return [coordinator.durableJobId, coordinator.fullJobId].some((jobId) => { + if (!jobId) return false; + const job = this.tracker.jobs.get(jobId); + return job?.status === 'queued' || job?.status === 'running'; + }); } private pruneCompletedJobs(): void { const activeJobIds = new Set(); for (const coordinator of this.inFlightByContextGraph.values()) { - for (const view of coordinator.viewsByScope.values()) { - activeJobIds.add(view.jobId); - } + if (coordinator.durableJobId) activeJobIds.add(coordinator.durableJobId); + if (coordinator.fullJobId) activeJobIds.add(coordinator.fullJobId); } while (this.tracker.jobs.size > 100) { let oldest: CatchupJob | undefined; @@ -211,37 +175,10 @@ export class ContextGraphCatchupCoordinatorService { readinessBeforeCatchup: ContextGraphReadinessProvenance, ): Promise { try { - for (let index = 0; index < coordinator.executions.length; index += 1) { - const execution = coordinator.executions[index]; - const job = this.tracker.jobs.get(execution.jobId); - if (!job || job.status !== 'queued') continue; - const readinessBeforeAttempt = index === 0 - ? readinessBeforeCatchup - : this.effects.readReadiness(coordinator.contextGraphId); - - try { - const attempt = await this.runAttempt(job, readinessBeforeAttempt); - await this.settleProjectionViews( - coordinator, - execution, - attempt.result, - readinessBeforeAttempt, - ); - if (attempt.status === 'denied') { - this.settleRemainingExecutionsFrom(coordinator, index + 1, job); - break; - } - } catch (error) { - job.error = error instanceof Error ? error.message : String(error); - job.status = 'failed'; - job.finishedAt = this.now(); - this.settleProjectionViewsFrom(coordinator, execution, job); - this.settleRemainingExecutionsFrom(coordinator, index + 1, job); - this.effects.trace?.( - `[catchup] job=${job.jobId} contextGraph=${coordinator.contextGraphId} threw: ${job.error}`, - ); - break; - } + if (coordinator.initialScope === 'durable') { + await this.runDurableFirst(coordinator, readinessBeforeCatchup); + } else { + await this.runFullFirst(coordinator, readinessBeforeCatchup); } } finally { if (this.inFlightByContextGraph.get(coordinator.contextGraphId) === coordinator) { @@ -250,10 +187,77 @@ export class ContextGraphCatchupCoordinatorService { } } + private async runDurableFirst( + coordinator: CatchupCoordinator, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + const durable = coordinator.durableJobId + ? this.tracker.jobs.get(coordinator.durableJobId) + : undefined; + if (!durable || durable.status !== 'queued') return; + try { + const attempt = await this.runAttempt(durable, readinessBeforeCatchup); + if (attempt.status === 'denied') { + this.settleFullSlotFrom(coordinator, durable); + return; + } + } catch (error) { + this.settleThrownJob(durable, error); + this.settleFullSlotFrom(coordinator, durable); + return; + } + + const full = coordinator.fullJobId + ? this.tracker.jobs.get(coordinator.fullJobId) + : undefined; + if (!full || full.status !== 'queued') return; + try { + await this.runAttempt(full, this.effects.readReadiness(coordinator.contextGraphId)); + } catch (error) { + this.settleThrownJob(full, error); + } + } + + private async runFullFirst( + coordinator: CatchupCoordinator, + readinessBeforeCatchup: ContextGraphReadinessProvenance, + ): Promise { + const full = coordinator.fullJobId + ? this.tracker.jobs.get(coordinator.fullJobId) + : undefined; + if (!full || full.status !== 'queued') return; + try { + const attempt = await this.runAttempt(full, readinessBeforeCatchup); + await this.settleDurableSlot( + coordinator, + full, + attempt.result, + readinessBeforeCatchup, + this.classificationHasEffects(attempt.classification), + ); + } catch (error) { + this.settleThrownJob(full, error); + this.settleDurableSlotFrom(coordinator, full); + } + } + + private settleThrownJob(job: CatchupJob, error: unknown): void { + job.error = error instanceof Error ? error.message : String(error); + job.status = 'failed'; + job.finishedAt = this.now(); + this.effects.trace?.( + `[catchup] job=${job.jobId} contextGraph=${job.contextGraphId} threw: ${job.error}`, + ); + } + private async runAttempt( job: CatchupJob, readinessBeforeCatchup: ContextGraphReadinessProvenance, - ): Promise<{ result: CatchupJobResult; status: CatchupJob['status'] }> { + ): Promise<{ + result: CatchupJobResult; + status: CatchupJob['status']; + classification: CatchupResultClassification; + }> { job.status = 'running'; job.startedAt ??= this.now(); this.effects.trace?.( @@ -270,7 +274,15 @@ export class ContextGraphCatchupCoordinatorService { ); this.applyExecutionEffects(job.contextGraphId, classification); this.settleClassifiedJob(job, result, classification); - return { result, status: job.status }; + return { result, status: job.status, classification }; + } + + private classificationHasEffects( + classification: CatchupResultClassification, + ): boolean { + return classification.readinessPatch !== undefined || + classification.statePatch !== undefined || + classification.eventPayload !== undefined; } private async classifyResult( @@ -278,43 +290,23 @@ export class ContextGraphCatchupCoordinatorService { result: CatchupJobResult, readinessBeforeCatchup: ContextGraphReadinessProvenance, ): Promise { - if (result.deferredBackpressure > 0 && !result.denied) { - return { - jobStatus: 'deferred' as const, - error: 'Sync deferred by local scheduler backpressure; retry when capacity is available.', - readinessPatch: undefined, - statePatch: undefined, - eventPayload: undefined, - }; - } - - const inspectReadiness = catchupResultHasCleanResponse(result); + const inspectReadiness = catchupClassificationNeedsMetadata({ + result, + includeSharedMemory: job.includeWorkspace, + }); const hasConfirmedMeta = inspectReadiness ? await this.effects.hasConfirmedMeta(job.contextGraphId) : false; const isPrivate = hasConfirmedMeta ? await this.effects.isPrivate(job.contextGraphId) : false; - const classification = classifyContextGraphCatchupReadiness({ + return classifyContextGraphCatchupReadiness({ result, includeSharedMemory: job.includeWorkspace, hasConfirmedMeta, isPrivate, readinessBeforeCatchup, }); - // Denial can coexist with usable data from another peer. If local - // admission also deferred part of that mixed round, the clean data must - // not turn the attempt into success: finalizeCatchup deliberately leaves - // any backpressured round incomplete. Preserve a pure ACL denial, but - // downgrade an otherwise-successful mixed result before effects are - // applied so no readiness bit is frozen from partial work. - if (result.deferredBackpressure > 0 && classification.jobStatus === 'done') { - return { - jobStatus: 'deferred', - error: 'Sync deferred by local scheduler backpressure; retry when capacity is available.', - }; - } - return classification; } private applyExecutionEffects( @@ -359,61 +351,52 @@ export class ContextGraphCatchupCoordinatorService { target.finishedAt = this.now(); } - private async settleProjectionViews( + private async settleDurableSlot( coordinator: CatchupCoordinator, - execution: CatchupExecution, + full: CatchupJob, result: CatchupJobResult, readinessBeforeCatchup: ContextGraphReadinessProvenance, + sourceAppliedEffects: boolean, ): Promise { - const source = this.tracker.jobs.get(execution.jobId); - if (!source) return; - for (const view of coordinator.viewsByScope.values()) { - if ( - view.kind !== 'projection' || - view.sourceExecutionJobId !== execution.jobId - ) continue; - const projection = this.tracker.jobs.get(view.jobId); - if (!projection || projection.status !== 'queued') continue; - projection.status = 'running'; - projection.startedAt = source.startedAt; - try { - const classification = await this.classifyResult( - projection, - result, - readinessBeforeCatchup, - ); - this.settleClassifiedJob(projection, result, classification); - } catch (error) { - projection.status = 'failed'; - projection.error = error instanceof Error ? error.message : String(error); - projection.finishedAt = this.now(); + const durable = coordinator.durableJobId + ? this.tracker.jobs.get(coordinator.durableJobId) + : undefined; + if (!durable || durable.status !== 'queued') return; + durable.status = 'running'; + durable.startedAt = full.startedAt; + try { + const classification = await this.classifyResult( + durable, + result, + readinessBeforeCatchup, + ); + // Full success already persisted every plane. When the full attempt had + // no effects (for example SWM-only local deferral), the independently + // successful durable slot owns its scoped readiness and event instead. + if (!sourceAppliedEffects) { + this.applyExecutionEffects(durable.contextGraphId, classification); } + this.settleClassifiedJob(durable, result, classification); + } catch (error) { + this.settleThrownJob(durable, error); } } - private settleProjectionViewsFrom( + private settleDurableSlotFrom( coordinator: CatchupCoordinator, - execution: CatchupExecution, source: CatchupJob, ): void { - for (const view of coordinator.viewsByScope.values()) { - if ( - view.kind === 'projection' && - view.sourceExecutionJobId === execution.jobId - ) { - this.settleQueuedJobFrom(view.jobId, source); - } + if (coordinator.durableJobId) { + this.settleQueuedJobFrom(coordinator.durableJobId, source); } } - private settleRemainingExecutionsFrom( + private settleFullSlotFrom( coordinator: CatchupCoordinator, - startIndex: number, source: CatchupJob, ): void { - for (const execution of coordinator.executions.slice(startIndex)) { - this.settleQueuedJobFrom(execution.jobId, source); - this.settleProjectionViewsFrom(coordinator, execution, source); + if (coordinator.fullJobId) { + this.settleQueuedJobFrom(coordinator.fullJobId, source); } } } diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 1969c8a15..a22da2c12 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -203,10 +203,12 @@ import { DkgClient } from '@origintrail-official/dkg-mcp/client'; // the project's tsconfig (`noUnusedLocals` is off). import { daemonState, + DEBUG_SYNC_TRACE, resolveStandaloneInstall, resolveAutoUpdatePollingMode, type CorsAllowlist, } from './state.js'; +import { createContextGraphCatchupRouteAdapter } from './context-graph-catchup-route-adapter.js'; import { type CatchupJobState, type CatchupJob, @@ -3351,7 +3353,6 @@ export async function runDaemonInner( const catchupTracker: CatchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - inFlightByContextGraph: new Map(), }; // --- Extraction Pipelines --- @@ -3460,6 +3461,15 @@ export async function runDaemonInner( let corsAllowed: CorsAllowlist = "*"; daemonState.catchupRunner = createCatchupRunner(agent); + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner, + readinessStore: dashDb, + agent, + ...(DEBUG_SYNC_TRACE + ? { trace: (message: string) => console.log(message) } + : {}), + }); const server = createServer(async (req, res) => { try { @@ -3655,6 +3665,7 @@ export async function runDaemonInner( nodeVersion, nodeCommit, catchupTracker, + catchupCoordinator, extractionRegistry, fileStore, extractionStatus, diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 9e96b02bd..12049e9f7 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -158,8 +158,6 @@ import { DkgClient } from '@origintrail-official/dkg-mcp/client'; // them all without explicit imports. Unused ones are tolerated by // the project's tsconfig (`noUnusedLocals` is off). import { - daemonState, - DEBUG_SYNC_TRACE, resolveAutoUpdateEnabled, type CorsAllowlist, } from '../state.js'; @@ -168,7 +166,6 @@ import { type CatchupJob, type CatchupTracker, } from '../types.js'; -import { createContextGraphCatchupRouteAdapter } from '../context-graph-catchup-route-adapter.js'; import { type MarkItDownTarget, manifestRepoRoot, @@ -481,6 +478,7 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise console.log(message) } - : {}), - }); const existingJobId = catchupTracker.latestByContextGraph.get(contextGraphId); const existingJob = existingJobId ? catchupTracker.jobs.get(existingJobId) : undefined; let readinessBeforeCatchup = readContextGraphReadiness(dashDb, contextGraphId); diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index dbad8dd72..bf3626947 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -24,6 +24,7 @@ import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; import type { CatchupTracker } from '../types.js'; import type { RoutePlugin } from '../plugin-api.js'; import type { AdmissionStatsView } from '../http-utils.js'; +import type { ContextGraphCatchupCoordinatorService } from '../context-graph-catchup-coordinator.js'; export type MemoryGraphLayer = 'wm' | 'swm' | 'vm'; @@ -73,6 +74,8 @@ export interface RequestContext { nodeVersion: string; nodeCommit: string; catchupTracker: CatchupTracker; + /** Lifecycle-owned coordinator; active coalescing state never leaks into status storage. */ + catchupCoordinator: ContextGraphCatchupCoordinatorService; extractionRegistry: ExtractionPipelineRegistry; fileStore: FileStore; extractionStatus: Map; diff --git a/packages/cli/src/daemon/types.ts b/packages/cli/src/daemon/types.ts index cde8556fe..c5b3639b8 100644 --- a/packages/cli/src/daemon/types.ts +++ b/packages/cli/src/daemon/types.ts @@ -21,44 +21,20 @@ export interface CatchupJob { export type CatchupScope = 'durable' | 'durable-and-shared-memory'; -export interface CatchupExecution { - jobId: string; - scope: CatchupScope; -} - -export type CatchupJobView = - | { - jobId: string; - scope: CatchupScope; - kind: 'execution'; - } - | { - jobId: string; - scope: CatchupScope; - sourceExecutionJobId: string; - kind: 'projection'; - }; - /** - * Mutable orchestration state for one serialized per-CG catch-up. Executions - * describe actual runner work; views describe the immutable public job for - * each requested scope. A narrow view can project a broad execution without - * pretending to be another execution, while a wider request queues one real - * serialized execution. + * One serialized per-CG catch-up has exactly the two scopes the product + * exposes. When durable starts first, a later full request occupies the full + * slot and runs second. When full starts first, a later durable request + * occupies the durable slot and is settled from that full result. */ export interface CatchupCoordinator { contextGraphId: string; - executions: CatchupExecution[]; - viewsByScope: Map; + initialScope: CatchupScope; + durableJobId?: string; + fullJobId?: string; } export interface CatchupTracker { jobs: Map; latestByContextGraph: Map; - /** - * Coordinator-only index added after the original two-map tracker contract. - * Optional at the public boundary so embedded callers can supply that legacy - * shape; the coordinator normalizes it through one canonical helper. - */ - inFlightByContextGraph?: Map; } diff --git a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts index 8bb42a8f3..693b1685d 100644 --- a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts +++ b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { privateDataOnlyResult, publicDurableAndSharedMemoryResult, + publicDurableWithSharedMemoryBackpressureResult, } from './helpers/context-graph-catchup-fixtures.js'; import { ContextGraphSubscribeRouteHarness } from './helpers/context-graph-subscribe-route-harness.js'; @@ -241,6 +242,50 @@ describe('context graph catch-up route coalescing', () => { } }); + it('keeps a VM projection done when only the broad SWM plane is backpressured', async () => { + const firstRunStarted = deferred(); + const releaseFirstRun = deferred(); + const harness = await ContextGraphSubscribeRouteHarness.create({ + hasConfirmedMeta: true, + initial: { + subscribed: false, + synced: false, + sharedMemorySynced: false, + metaSynced: true, + }, + runner: async () => { + firstRunStarted.resolve(); + await releaseFirstRun.promise; + return publicDurableWithSharedMemoryBackpressureResult(); + }, + }); + + try { + const broad = await harness.postSubscribe({ includeSharedMemory: true }); + await firstRunStarted.promise; + const narrow = await harness.postSubscribe({ includeSharedMemory: false }); + + releaseFirstRun.resolve(); + const broadJob = await harness.waitForJob(broad.body.catchup.jobId); + const narrowJob = await harness.waitForJob(narrow.body.catchup.jobId); + + expect(harness.runCalls).toBe(1); + expect(broadJob).toMatchObject({ status: 'deferred' }); + expect(narrowJob).toMatchObject({ status: 'done' }); + await expect(harness.getStatus(narrow.body.catchup.jobId)).resolves.toMatchObject({ + status: 'done', + convergence: { + state: 'complete', + required: { sharedMemory: false }, + missing: [], + }, + }); + } finally { + releaseFirstRun.resolve(); + await harness.close(); + } + }); + it('keeps VM-only success stable when the serialized SWM upgrade fails', async () => { const firstRunStarted = deferred(); const releaseFirstRun = deferred(); diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index 5b1c558a7..a3dbce35e 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -7,6 +7,7 @@ import { cleanEmptyResult, privateDataOnlyResult, publicDurableAndSharedMemoryResult, + publicDurableWithSharedMemoryBackpressureResult, } from './helpers/context-graph-catchup-fixtures.js'; function deferred(): { promise: Promise; resolve: () => void } { @@ -48,7 +49,6 @@ function coordinatorFixture(options: { const tracker: CatchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - inFlightByContextGraph: new Map(), }; let readiness: ContextGraphReadinessProvenance = { version: 1, @@ -62,7 +62,6 @@ function coordinatorFixture(options: { let runNumber = 0; const run = vi.fn(async (request: { includeSharedMemory: boolean }) => { runNumber += 1; - if (options.result) return options.result; if (runNumber === 1 && (!request.includeSharedMemory || options.blockBroadBase)) { firstRunStarted.resolve(); await releaseFirstRun.promise; @@ -70,8 +69,10 @@ function coordinatorFixture(options: { throw new Error('base attempt failed'); } if (options.baseOutcome === 'denied') return deniedResult(); + if (options.result) return options.result; return privateDataOnlyResult(); } + if (options.result) return options.result; return options.failUpgrade ? privateDataOnlyResult() : publicDurableAndSharedMemoryResult(); @@ -105,6 +106,48 @@ function coordinatorFixture(options: { } describe('ContextGraphCatchupCoordinatorService', () => { + it('settles a durable slot from broad work without inheriting SWM backpressure', async () => { + const fixture = coordinatorFixture({ + blockBroadBase: true, + result: publicDurableWithSharedMemoryBackpressureResult(), + }); + const full = fixture.service.start({ + contextGraphId: 'cg:scope-backpressure', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + await fixture.firstRunStarted.promise; + const durable = fixture.service.coalesceActive({ + contextGraphId: 'cg:scope-backpressure', + includeSharedMemory: false, + }); + + fixture.releaseFirstRun.resolve(); + await waitForJob(full); + if (!durable) throw new Error('durable slot missing'); + await waitForJob(durable); + + expect(fixture.run).toHaveBeenCalledTimes(1); + expect(full).toMatchObject({ + includeWorkspace: true, + status: 'deferred', + }); + expect(durable).toMatchObject({ + includeWorkspace: false, + status: 'done', + }); + expect(fixture.writeReadiness).toHaveBeenCalledTimes(1); + expect(fixture.writeReadiness).toHaveBeenCalledWith( + 'cg:scope-backpressure', + { durableVerified: true, sharedMemoryVerified: false }, + ); + }); + it('keeps denied mixed progress deferred when local backpressure left the round incomplete', async () => { const result = publicDurableAndSharedMemoryResult(); result.denied = true; @@ -132,9 +175,7 @@ describe('ContextGraphCatchupCoordinatorService', () => { expect(fixture.markSubscriptionState).not.toHaveBeenCalled(); }); - it('provisions orchestration state for the historical two-map tracker shape', () => { - // Compile-time regression: the public tracker boundary explicitly accepts - // the historical two-map shape without a cast. + it('keeps orchestration state out of the historical two-map tracker', () => { const tracker: CatchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), @@ -153,7 +194,7 @@ describe('ContextGraphCatchupCoordinatorService', () => { contextGraphId: 'cg:legacy-tracker', includeSharedMemory: true, })).toBeUndefined(); - expect(tracker.inFlightByContextGraph).toBeInstanceOf(Map); + expect(Object.keys(tracker).sort()).toEqual(['jobs', 'latestByContextGraph']); }); it('refreshes latest status when broad, narrow, then broad reuses existing views', async () => { @@ -265,7 +306,10 @@ describe('ContextGraphCatchupCoordinatorService', () => { ]); expect(base).toMatchObject({ includeWorkspace: false, status: 'done' }); expect(upgrade).toMatchObject({ includeWorkspace: true, status: 'done' }); - expect(fixture.tracker.inFlightByContextGraph?.has('cg:one')).toBe(false); + expect(fixture.service.coalesceActive({ + contextGraphId: 'cg:one', + includeSharedMemory: true, + })).toBeUndefined(); }); it('does not retroactively fail VM-only success when the wider upgrade is incomplete', async () => { diff --git a/packages/cli/test/context-graph-catchup-readiness.test.ts b/packages/cli/test/context-graph-catchup-readiness.test.ts index cd0e070df..1c36caf8f 100644 --- a/packages/cli/test/context-graph-catchup-readiness.test.ts +++ b/packages/cli/test/context-graph-catchup-readiness.test.ts @@ -6,6 +6,10 @@ import { combineCatchupPlaneEvidence, describeContextGraphConvergence, } from '../src/context-graph-readiness.js'; +import { + publicDurableAndSharedMemoryResult, + publicDurableWithSharedMemoryBackpressureResult, +} from './helpers/context-graph-catchup-fixtures.js'; function mixedPeerResult(verifiedDataPeers: number): CatchupJobResult { return { @@ -72,6 +76,44 @@ describe('context graph catch-up readiness classification', () => { updatedAt: 0, }; + it('scopes shared-memory backpressure out of a durable-only view', () => { + const result = publicDurableWithSharedMemoryBackpressureResult(); + const classify = (includeSharedMemory: boolean) => + classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory, + hasConfirmedMeta: true, + isPrivate: false, + readinessBeforeCatchup, + }); + + expect(classify(false)).toMatchObject({ jobStatus: 'done' }); + expect(classify(true)).toMatchObject({ + jobStatus: 'deferred', + error: expect.stringContaining('local scheduler backpressure'), + }); + }); + + it('keeps durable-plane backpressure deferred for narrow and broad views', () => { + const result = publicDurableAndSharedMemoryResult(); + result.deferredBackpressure = 1; + if (!result.diagnostics) throw new Error('diagnostics missing'); + result.diagnostics.durable.deferredBackpressure = 1; + + for (const includeSharedMemory of [false, true]) { + expect(classifyContextGraphCatchupReadiness({ + result, + includeSharedMemory, + hasConfirmedMeta: true, + isPrivate: false, + readinessBeforeCatchup, + })).toMatchObject({ + jobStatus: 'deferred', + error: expect.stringContaining('local scheduler backpressure'), + }); + } + }); + it('uses a clean per-peer completion even when aggregate diagnostics contain denial and timeout', () => { const classification = classifyContextGraphCatchupReadiness({ result: mixedPeerResult(1), diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index a0cc0e47d..5eb50800b 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -57,6 +57,7 @@ import { getSharedContext, HARDHAT_KEYS } from '../../chain/test/evm-test-contex import { ApiClient } from '../src/api-client.js'; import { handleContextGraphRoutes } from '../src/daemon/routes/context-graph.js'; import { daemonState } from '../src/daemon/state.js'; +import { createContextGraphCatchupRouteAdapter } from '../src/daemon/context-graph-catchup-route-adapter.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const CLI_ENTRY = join(__dirname, '..', 'dist', 'cli.js'); @@ -997,7 +998,6 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - inFlightByContextGraph: new Map(), }; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { @@ -1067,6 +1067,12 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { resolveAgentByToken: () => undefined, getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, res, @@ -1084,6 +1090,7 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), @@ -1151,7 +1158,6 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - inFlightByContextGraph: new Map(), }; const previousCatchupRunner = daemonState.catchupRunner; daemonState.catchupRunner = { @@ -1188,6 +1194,12 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { resolveAgentByToken: () => undefined, getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, res, @@ -1205,6 +1217,7 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), @@ -1304,6 +1317,12 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { resolveAgentByToken: () => undefined, getDefaultAgentAddress: () => '0x0000000000000000000000000000000000000001', }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, @@ -1322,6 +1341,7 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), @@ -2411,7 +2431,6 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli const catchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - inFlightByContextGraph: new Map(), }; const previousCatchupRunner = daemonState.catchupRunner; // Benign runner: the queued job runs fire-and-forget after the response and @@ -2457,6 +2476,12 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli markContextGraphSubscriptionState: () => {}, resolveAgentByToken: () => undefined, }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: catchupTracker, + runner: daemonState.catchupRunner!, + readinessStore: {} as any, + agent: agent as any, + }); await handleContextGraphRoutes({ req, res, @@ -2474,6 +2499,7 @@ describe('#1596 — subscribe allowlist gate respects explicit public accessPoli nodeVersion: 'test', nodeCommit: 'test', catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), diff --git a/packages/cli/test/helpers/context-graph-catchup-fixtures.ts b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts index a18e79394..c686854e9 100644 --- a/packages/cli/test/helpers/context-graph-catchup-fixtures.ts +++ b/packages/cli/test/helpers/context-graph-catchup-fixtures.ts @@ -197,6 +197,30 @@ export function publicDurableAndSharedMemoryResult(): CatchupJobResult { }); } +/** Durable VM completed, but the shared-memory scheduler deferred its plane. */ +export function publicDurableWithSharedMemoryBackpressureResult(): CatchupJobResult { + return makeCatchupJobResult({ + deferredBackpressure: 1, + dataSynced: 3, + cleanPlaneCompletions: { + durable: { verifiedDataPeers: 1, emptyPeers: 0 }, + sharedMemory: { verifiedDataPeers: 0, emptyPeers: 0 }, + }, + diagnostics: { + durable: { + emptyResponses: 0, + fetchedDataTriples: 3, + insertedDataTriples: 3, + }, + sharedMemory: { + emptyResponses: 0, + completedPhases: 0, + deferredBackpressure: 1, + }, + }, + }); +} + /** One public host proves empty while another selected peer transport-fails. */ export function lossyPublicEmptyResult(): CatchupJobResult { return makeCatchupJobResult({ diff --git a/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts index 2ec75858f..3ffd5c21f 100644 --- a/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts +++ b/packages/cli/test/helpers/context-graph-subscribe-route-harness.ts @@ -7,6 +7,7 @@ import type { import { handleContextGraphRoutes } from '../../src/daemon/routes/context-graph.js'; import { handleQueryRoutes } from '../../src/daemon/routes/query.js'; import { daemonState } from '../../src/daemon/state.js'; +import { createContextGraphCatchupRouteAdapter } from '../../src/daemon/context-graph-catchup-route-adapter.js'; import type { CatchupJob } from '../../src/daemon/types.js'; import { cleanEmptyResult } from './context-graph-catchup-fixtures.js'; @@ -54,7 +55,6 @@ export class ContextGraphSubscribeRouteHarness { private readonly catchupTracker = { jobs: new Map(), latestByContextGraph: new Map(), - inFlightByContextGraph: new Map(), }; private server: Server | undefined; private addressPort = 0; @@ -232,6 +232,26 @@ export class ContextGraphSubscribeRouteHarness { this.options.callerAddress ?? '0x0000000000000000000000000000000000000001', }; + const readinessStore = { + getContextGraphReadinessProvenance: () => this.readinessValue ?? null, + setContextGraphReadinessProvenance: ( + _id: string, + next: { + version: number; + durableVerified: boolean; + sharedMemoryVerified: boolean; + }, + ) => { + this.readinessValue = { ...next, updatedAt: Date.now() }; + }, + }; + const catchupCoordinator = createContextGraphCatchupRouteAdapter({ + tracker: this.catchupTracker, + runner: daemonState.catchupRunner, + readinessStore: readinessStore as any, + agent: agent as any, + }); + this.server = createServer(async (request, response) => { const url = new URL(request.url ?? '/', 'http://127.0.0.1'); const routeContext = { @@ -242,19 +262,7 @@ export class ContextGraphSubscribeRouteHarness { publisherRuntime: null, config: { auth: { enabled: false } }, startedAt: Date.now(), - dashDb: { - getContextGraphReadinessProvenance: () => this.readinessValue ?? null, - setContextGraphReadinessProvenance: ( - _id: string, - next: { - version: number; - durableVerified: boolean; - sharedMemoryVerified: boolean; - }, - ) => { - this.readinessValue = { ...next, updatedAt: Date.now() }; - }, - }, + dashDb: readinessStore, opWallets: {}, network: {}, tracker: {}, @@ -263,6 +271,7 @@ export class ContextGraphSubscribeRouteHarness { nodeVersion: 'test', nodeCommit: 'test', catchupTracker: this.catchupTracker, + catchupCoordinator, extractionRegistry: {}, fileStore: {}, extractionStatus: new Map(), From 0427257ae6b54caecc1dd5e8815521a548e2fdde Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 17:07:43 +0200 Subject: [PATCH 22/23] refactor(sync): canonicalize shared-memory scope --- packages/cli/src/api-client.ts | 6 +++++- packages/cli/src/catchup-status-wire.ts | 1 + packages/cli/src/cli-helpers.ts | 2 +- .../cli/src/daemon/catchup-status-response.ts | 5 +++-- .../context-graph-catchup-coordinator.ts | 8 ++++---- .../cli/src/daemon/routes/context-graph.ts | 9 ++++++--- packages/cli/src/daemon/types.ts | 2 +- packages/cli/test/catchup-status-cli.test.ts | 3 ++- .../cli/test/catchup-status-response.test.ts | 13 ++++++++++++- ...text-graph-catchup-coalescing-route.test.ts | 18 +++++++++--------- .../context-graph-catchup-coordinator.test.ts | 16 ++++++++-------- 11 files changed, 52 insertions(+), 31 deletions(-) diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index 2c585c3f4..5557b55db 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -1554,13 +1554,15 @@ export class ApiClient { } | { status: 'queued'; + includeSharedMemory: boolean; + /** @deprecated Backward-compatible response alias. */ includeWorkspace: boolean; jobId: string; }; }> { return this.post('/api/context-graph/subscribe', { contextGraphId, - includeWorkspace: options.includeSharedMemory, + includeSharedMemory: options.includeSharedMemory, syncMode: options.syncMode, }); } @@ -1634,6 +1636,8 @@ export class ApiClient { } | { status: 'queued'; + includeSharedMemory: boolean; + /** @deprecated Backward-compatible response alias. */ includeWorkspace: boolean; jobId: string; }; diff --git a/packages/cli/src/catchup-status-wire.ts b/packages/cli/src/catchup-status-wire.ts index 5570f8fe0..97b1748d6 100644 --- a/packages/cli/src/catchup-status-wire.ts +++ b/packages/cli/src/catchup-status-wire.ts @@ -24,6 +24,7 @@ export interface CatchupConvergenceStatus extends ContextGraphConvergenceSnapsho export interface CatchupStatusResponse { jobId: string; contextGraphId: string; + /** @deprecated Backward-compatible response alias for includeSharedMemory. */ includeWorkspace: boolean; includeSharedMemory: boolean; /** Canonical actionable status, including newer live convergence evidence. */ diff --git a/packages/cli/src/cli-helpers.ts b/packages/cli/src/cli-helpers.ts index f09b86ffa..98ac1c923 100644 --- a/packages/cli/src/cli-helpers.ts +++ b/packages/cli/src/cli-helpers.ts @@ -206,7 +206,7 @@ function printCatchupStatus(status: Awaited { const inspectReadiness = catchupClassificationNeedsMetadata({ result, - includeSharedMemory: job.includeWorkspace, + includeSharedMemory: job.includeSharedMemory, }); const hasConfirmedMeta = inspectReadiness ? await this.effects.hasConfirmedMeta(job.contextGraphId) @@ -302,7 +302,7 @@ export class ContextGraphCatchupCoordinatorService { : false; return classifyContextGraphCatchupReadiness({ result, - includeSharedMemory: job.includeWorkspace, + includeSharedMemory: job.includeSharedMemory, hasConfirmedMeta, isPrivate, readinessBeforeCatchup, diff --git a/packages/cli/src/daemon/routes/context-graph.ts b/packages/cli/src/daemon/routes/context-graph.ts index 12049e9f7..a898cfa72 100644 --- a/packages/cli/src/daemon/routes/context-graph.ts +++ b/packages/cli/src/daemon/routes/context-graph.ts @@ -1753,7 +1753,8 @@ export async function handleContextGraphRoutes(ctx: RequestContext): Promise { expect(lines).toContain('Status: done'); expect(lines).toContain('Last attempt: unreachable'); expect(lines).toContain('Attempt error: durable VM missing'); + expect(lines).toContain('Shared Memory: enabled'); expect(lines).toContain('Convergence: complete'); expect(lines).toContain('Recovered: a later synchronization completed the selected graph'); }); diff --git a/packages/cli/test/catchup-status-response.test.ts b/packages/cli/test/catchup-status-response.test.ts index 47ab9faf9..1c4f54038 100644 --- a/packages/cli/test/catchup-status-response.test.ts +++ b/packages/cli/test/catchup-status-response.test.ts @@ -28,7 +28,7 @@ function job(status: CatchupJob['status']): CatchupJob { return { jobId: 'job-1', contextGraphId: 'cg-1', - includeWorkspace: true, + includeSharedMemory: true, status, queuedAt: 1, startedAt: 2, @@ -38,6 +38,17 @@ function job(status: CatchupJob['status']): CatchupJob { } describe('catch-up status response', () => { + it('keeps the legacy workspace name at the response boundary only', () => { + const attempt = job('done'); + + expect(attempt).toHaveProperty('includeSharedMemory', true); + expect(attempt).not.toHaveProperty('includeWorkspace'); + expect(toCatchupStatusResponse(attempt)).toMatchObject({ + includeSharedMemory: true, + includeWorkspace: true, + }); + }); + it('reports live completion while preserving a failed attempt as diagnostics', () => { expect(toCatchupStatusResponse(job('failed'), completeConvergence)).toMatchObject({ status: 'done', diff --git a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts index 693b1685d..118f6c68a 100644 --- a/packages/cli/test/context-graph-catchup-coalescing-route.test.ts +++ b/packages/cli/test/context-graph-catchup-coalescing-route.test.ts @@ -127,7 +127,7 @@ describe('context graph catch-up route coalescing', () => { expect(upgrade.body.catchup).toMatchObject({ status: 'queued', - includeWorkspace: true, + includeSharedMemory: true, }); expect(upgrade.body.catchup.jobId).not.toBe(base.body.catchup.jobId); expect(harness.runCalls).toBe(1); @@ -135,8 +135,8 @@ describe('context graph catch-up route coalescing', () => { await expect(harness.getStatusByContextGraph()).resolves.toMatchObject({ jobId: upgrade.body.catchup.jobId, status: 'queued', - includeWorkspace: true, includeSharedMemory: true, + includeWorkspace: true, }); releaseFirstRun.resolve(); @@ -153,8 +153,8 @@ describe('context graph catch-up route coalescing', () => { includeSharedMemory: true, }, ]); - expect(baseJob).toMatchObject({ includeWorkspace: false, status: 'done' }); - expect(upgradeJob).toMatchObject({ includeWorkspace: true, status: 'done' }); + expect(baseJob).toMatchObject({ includeSharedMemory: false, status: 'done' }); + expect(upgradeJob).toMatchObject({ includeSharedMemory: true, status: 'done' }); await expect(harness.getStatus(base.body.catchup.jobId)).resolves.toMatchObject({ status: 'done', convergence: { @@ -202,7 +202,7 @@ describe('context graph catch-up route coalescing', () => { expect(narrow.body.catchup).toMatchObject({ status: 'queued', - includeWorkspace: false, + includeSharedMemory: false, }); expect(narrow.body.catchup.jobId).not.toBe(broad.body.catchup.jobId); expect(harness.runCalls).toBe(1); @@ -213,11 +213,11 @@ describe('context graph catch-up route coalescing', () => { expect(harness.runCalls).toBe(1); expect(broadJob).toMatchObject({ - includeWorkspace: true, + includeSharedMemory: true, status: 'unreachable', }); expect(narrowJob).toMatchObject({ - includeWorkspace: false, + includeSharedMemory: false, status: 'done', }); await expect(harness.getStatus(narrow.body.catchup.jobId)).resolves.toMatchObject({ @@ -314,9 +314,9 @@ describe('context graph catch-up route coalescing', () => { const baseJob = await harness.waitForJob(base.body.catchup.jobId); const upgradeJob = await harness.waitForJob(upgrade.body.catchup.jobId); - expect(baseJob).toMatchObject({ includeWorkspace: false, status: 'done' }); + expect(baseJob).toMatchObject({ includeSharedMemory: false, status: 'done' }); expect(upgradeJob).toMatchObject({ - includeWorkspace: true, + includeSharedMemory: true, status: 'unreachable', error: expect.stringContaining('requested data plane'), }); diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index a3dbce35e..a46a46d8a 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -134,11 +134,11 @@ describe('ContextGraphCatchupCoordinatorService', () => { expect(fixture.run).toHaveBeenCalledTimes(1); expect(full).toMatchObject({ - includeWorkspace: true, + includeSharedMemory: true, status: 'deferred', }); expect(durable).toMatchObject({ - includeWorkspace: false, + includeSharedMemory: false, status: 'done', }); expect(fixture.writeReadiness).toHaveBeenCalledTimes(1); @@ -290,8 +290,8 @@ describe('ContextGraphCatchupCoordinatorService', () => { includeSharedMemory: true, }); - expect(base).toMatchObject({ jobId: 'job-1', includeWorkspace: false }); - expect(upgrade).toMatchObject({ jobId: 'job-2', includeWorkspace: true }); + expect(base).toMatchObject({ jobId: 'job-1', includeSharedMemory: false }); + expect(upgrade).toMatchObject({ jobId: 'job-2', includeSharedMemory: true }); expect(repeatedUpgrade?.jobId).toBe(upgrade?.jobId); expect(fixture.run).toHaveBeenCalledTimes(1); @@ -304,8 +304,8 @@ describe('ContextGraphCatchupCoordinatorService', () => { { contextGraphId: 'cg:one', includeSharedMemory: false }, { contextGraphId: 'cg:one', includeSharedMemory: true }, ]); - expect(base).toMatchObject({ includeWorkspace: false, status: 'done' }); - expect(upgrade).toMatchObject({ includeWorkspace: true, status: 'done' }); + expect(base).toMatchObject({ includeSharedMemory: false, status: 'done' }); + expect(upgrade).toMatchObject({ includeSharedMemory: true, status: 'done' }); expect(fixture.service.coalesceActive({ contextGraphId: 'cg:one', includeSharedMemory: true, @@ -335,9 +335,9 @@ describe('ContextGraphCatchupCoordinatorService', () => { if (!upgrade) throw new Error('upgrade job missing'); await waitForJob(upgrade); - expect(base).toMatchObject({ includeWorkspace: false, status: 'done' }); + expect(base).toMatchObject({ includeSharedMemory: false, status: 'done' }); expect(upgrade).toMatchObject({ - includeWorkspace: true, + includeSharedMemory: true, status: 'unreachable', error: expect.stringContaining('requested data plane'), }); From da58c0c70453b32af4f355b00ca2c7a8fdf78fa9 Mon Sep 17 00:00:00 2001 From: Branimir Rakic Date: Sun, 2 Aug 2026 17:41:21 +0200 Subject: [PATCH 23/23] refactor(sync): own metadata classification policy --- packages/cli/src/context-graph-readiness.ts | 82 ++++++++++++------- .../context-graph-catchup-coordinator.ts | 20 ++--- packages/cli/test/api-client.test.ts | 6 +- .../cli/test/catchup-status-response.test.ts | 14 ++++ .../context-graph-catchup-coordinator.test.ts | 34 +++++++- .../context-graph-catchup-readiness.test.ts | 15 +++- 6 files changed, 122 insertions(+), 49 deletions(-) diff --git a/packages/cli/src/context-graph-readiness.ts b/packages/cli/src/context-graph-readiness.ts index e8ab20425..5050f2390 100644 --- a/packages/cli/src/context-graph-readiness.ts +++ b/packages/cli/src/context-graph-readiness.ts @@ -492,49 +492,71 @@ function requestedCatchupBackpressure( ?? result.deferredBackpressure; } -/** Whether the canonical classifier needs chain-backed CG metadata this round. */ -export function catchupClassificationNeedsMetadata(input: { +interface ContextGraphCatchupReadinessInput { result: CatchupJobResult; includeSharedMemory: boolean; -}): boolean { - const deferredBackpressure = requestedCatchupBackpressure( - input.result, - input.includeSharedMemory, - ); - // A pure local deferral settles before any metadata/readiness branch. Mixed - // denial still needs the full classifier to distinguish denial from usable - // progress, so it must inspect clean evidence normally. - if (deferredBackpressure > 0 && !input.result.denied) return false; - return catchupResultHasCleanResponse(input.result); + readinessBeforeCatchup: ContextGraphReadinessProvenance; } -/** - * Canonical policy for converting one catch-up result into externally visible - * subscription readiness. The HTTP route gathers live metadata and applies - * the returned patches; all readiness decisions remain in this pure function. - */ -export function classifyContextGraphCatchupReadiness(input: { - result: CatchupJobResult; - includeSharedMemory: boolean; +export interface ContextGraphCatchupMetadata { hasConfirmedMeta: boolean; isPrivate: boolean; - readinessBeforeCatchup: ContextGraphReadinessProvenance; -}): ContextGraphCatchupReadinessClassification { +} + +export type ContextGraphCatchupReadinessPlan = + | { + kind: 'settled'; + classification: ContextGraphCatchupReadinessClassification; + } + | { + kind: 'metadata-required'; + finalize: ( + metadata: ContextGraphCatchupMetadata, + ) => ContextGraphCatchupReadinessClassification; + }; + +/** + * Canonical two-phase classifier for one catch-up attempt. The plan owns the + * decision to request chain-backed metadata, so orchestration never mirrors + * classifier branches merely to avoid an unnecessary metadata read. + */ +export function planContextGraphCatchupReadiness( + input: ContextGraphCatchupReadinessInput, +): ContextGraphCatchupReadinessPlan { const deferredBackpressure = requestedCatchupBackpressure( input.result, input.includeSharedMemory, ); if (deferredBackpressure > 0 && !input.result.denied) { - return deferredCatchupClassification(); + return { + kind: 'settled', + classification: deferredCatchupClassification(), + }; + } + + const finalize = ( + metadata: ContextGraphCatchupMetadata, + ): ContextGraphCatchupReadinessClassification => { + const classification = classifyContextGraphCatchupReadinessWithoutBackpressure({ + ...input, + ...metadata, + }); + // Pure ACL denial keeps its specific status. When denial coexists with + // clean requested-plane progress, local deferral still prevents that + // partial round from becoming successful or freezing readiness. + return deferredBackpressure > 0 && classification.jobStatus === 'done' + ? deferredCatchupClassification() + : classification; + }; + + if (!catchupResultHasCleanResponse(input.result)) { + return { + kind: 'settled', + classification: finalize({ hasConfirmedMeta: false, isPrivate: false }), + }; } - const classification = classifyContextGraphCatchupReadinessWithoutBackpressure(input); - // Pure ACL denial keeps its specific status. When denial coexists with clean - // requested-plane progress, local deferral still prevents that partial round - // from becoming successful or freezing readiness. - return deferredBackpressure > 0 && classification.jobStatus === 'done' - ? deferredCatchupClassification() - : classification; + return { kind: 'metadata-required', finalize }; } function classifyContextGraphCatchupReadinessWithoutBackpressure(input: { diff --git a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts index 2415a9988..7a9f68f90 100644 --- a/packages/cli/src/daemon/context-graph-catchup-coordinator.ts +++ b/packages/cli/src/daemon/context-graph-catchup-coordinator.ts @@ -2,8 +2,7 @@ import type { ContextGraphReadinessProvenance } from '@origintrail-official/dkg- import type { CatchupRunner } from '../catchup-runner.js'; import type { CatchupJobResult } from '../catchup-result-wire.js'; import { - catchupClassificationNeedsMetadata, - classifyContextGraphCatchupReadiness, + planContextGraphCatchupReadiness, type ContextGraphCatchupReadinessClassification, } from '../context-graph-readiness.js'; import type { @@ -290,23 +289,18 @@ export class ContextGraphCatchupCoordinatorService { result: CatchupJobResult, readinessBeforeCatchup: ContextGraphReadinessProvenance, ): Promise { - const inspectReadiness = catchupClassificationNeedsMetadata({ + const plan = planContextGraphCatchupReadiness({ result, includeSharedMemory: job.includeSharedMemory, + readinessBeforeCatchup, }); - const hasConfirmedMeta = inspectReadiness - ? await this.effects.hasConfirmedMeta(job.contextGraphId) - : false; + if (plan.kind === 'settled') return plan.classification; + + const hasConfirmedMeta = await this.effects.hasConfirmedMeta(job.contextGraphId); const isPrivate = hasConfirmedMeta ? await this.effects.isPrivate(job.contextGraphId) : false; - return classifyContextGraphCatchupReadiness({ - result, - includeSharedMemory: job.includeSharedMemory, - hasConfirmedMeta, - isPrivate, - readinessBeforeCatchup, - }); + return plan.finalize({ hasConfirmedMeta, isPrivate }); } private applyExecutionEffects( diff --git a/packages/cli/test/api-client.test.ts b/packages/cli/test/api-client.test.ts index 22b5de629..0d64d8aaa 100644 --- a/packages/cli/test/api-client.test.ts +++ b/packages/cli/test/api-client.test.ts @@ -475,12 +475,12 @@ describe('ApiClient', () => { expect(calls[0].url).toBe(`http://127.0.0.1:${PORT}/api/context-graph/subscribe`); expect(JSON.parse(calls[0].opts.body as string)).toEqual({ contextGraphId: 'cg-selected', - includeWorkspace: true, + includeSharedMemory: true, syncMode: 'on-demand', }); }); - it('subscribe() keeps the legacy restart-durable lifetime explicit', async () => { + it('subscribe() maps its deprecated workspace option to the canonical request key', async () => { const { fetch, calls } = createTrackingFetch({ ok: true, status: 200, @@ -492,7 +492,7 @@ describe('ApiClient', () => { expect(JSON.parse(calls[0].opts.body as string)).toEqual({ contextGraphId: 'cg-legacy', - includeWorkspace: true, + includeSharedMemory: true, syncMode: 'always-on', }); }); diff --git a/packages/cli/test/catchup-status-response.test.ts b/packages/cli/test/catchup-status-response.test.ts index 1c4f54038..25ff76f54 100644 --- a/packages/cli/test/catchup-status-response.test.ts +++ b/packages/cli/test/catchup-status-response.test.ts @@ -63,6 +63,20 @@ describe('catch-up status response', () => { .not.toHaveProperty('error'); }); + it('reports newer live completion while preserving a deferred attempt as diagnostics', () => { + expect(toCatchupStatusResponse(job('deferred'), completeConvergence)).toMatchObject({ + status: 'done', + attempt: { + status: 'deferred', + error: 'deferred attempt', + }, + completedAfterAttempt: true, + convergence: completeConvergence, + }); + expect(toCatchupStatusResponse(job('deferred'), completeConvergence)) + .not.toHaveProperty('error'); + }); + it('does not hide a failed attempt behind readiness that predates it', () => { const staleConvergence = { ...completeConvergence, readinessUpdatedAt: 3 }; diff --git a/packages/cli/test/context-graph-catchup-coordinator.test.ts b/packages/cli/test/context-graph-catchup-coordinator.test.ts index a46a46d8a..bf52629a0 100644 --- a/packages/cli/test/context-graph-catchup-coordinator.test.ts +++ b/packages/cli/test/context-graph-catchup-coordinator.test.ts @@ -84,11 +84,13 @@ function coordinatorFixture(options: { readiness = { ...readiness, ...patch, updatedAt: readiness.updatedAt + 1 }; }); const markSubscriptionState = vi.fn(); + const hasConfirmedMeta = vi.fn(async () => true); + const isPrivate = vi.fn(async () => false); const service = new ContextGraphCatchupCoordinatorService(tracker, { runner: { run }, readReadiness: () => readiness, - hasConfirmedMeta: async () => true, - isPrivate: async () => false, + hasConfirmedMeta, + isPrivate, writeReadiness, markSubscriptionState, emitProjectSynced: vi.fn(), @@ -102,10 +104,34 @@ function coordinatorFixture(options: { releaseFirstRun, writeReadiness, markSubscriptionState, + hasConfirmedMeta, + isPrivate, }; } describe('ContextGraphCatchupCoordinatorService', () => { + it('does not load metadata for pure local deferral', async () => { + const fixture = coordinatorFixture({ + result: publicDurableWithSharedMemoryBackpressureResult(), + }); + const full = fixture.service.start({ + contextGraphId: 'cg:pure-deferral', + includeSharedMemory: true, + readinessBeforeCatchup: { + version: 1, + durableVerified: false, + sharedMemoryVerified: false, + updatedAt: 1, + }, + }); + + await waitForJob(full); + + expect(full.status).toBe('deferred'); + expect(fixture.hasConfirmedMeta).not.toHaveBeenCalled(); + expect(fixture.isPrivate).not.toHaveBeenCalled(); + }); + it('settles a durable slot from broad work without inheriting SWM backpressure', async () => { const fixture = coordinatorFixture({ blockBroadBase: true, @@ -173,6 +199,8 @@ describe('ContextGraphCatchupCoordinatorService', () => { }); expect(fixture.writeReadiness).not.toHaveBeenCalled(); expect(fixture.markSubscriptionState).not.toHaveBeenCalled(); + expect(fixture.hasConfirmedMeta).toHaveBeenCalledTimes(1); + expect(fixture.isPrivate).toHaveBeenCalledTimes(1); }); it('keeps orchestration state out of the historical two-map tracker', () => { @@ -404,5 +432,7 @@ describe('ContextGraphCatchupCoordinatorService', () => { status: 'denied', finishedAt: expect.any(Number), }); + expect(fixture.hasConfirmedMeta).not.toHaveBeenCalled(); + expect(fixture.isPrivate).not.toHaveBeenCalled(); }); }); diff --git a/packages/cli/test/context-graph-catchup-readiness.test.ts b/packages/cli/test/context-graph-catchup-readiness.test.ts index 1c36caf8f..cea03f237 100644 --- a/packages/cli/test/context-graph-catchup-readiness.test.ts +++ b/packages/cli/test/context-graph-catchup-readiness.test.ts @@ -2,9 +2,9 @@ import { describe, expect, it } from 'vitest'; import type { CatchupJobResult } from '../src/catchup-runner.js'; import { CONTEXT_GRAPH_READINESS_VERSION, - classifyContextGraphCatchupReadiness, combineCatchupPlaneEvidence, describeContextGraphConvergence, + planContextGraphCatchupReadiness, } from '../src/context-graph-readiness.js'; import { publicDurableAndSharedMemoryResult, @@ -68,6 +68,19 @@ function mixedPeerResult(verifiedDataPeers: number): CatchupJobResult { }; } +function classifyContextGraphCatchupReadiness( + input: Parameters[0] & { + hasConfirmedMeta: boolean; + isPrivate: boolean; + }, +) { + const { hasConfirmedMeta, isPrivate, ...planInput } = input; + const plan = planContextGraphCatchupReadiness(planInput); + return plan.kind === 'settled' + ? plan.classification + : plan.finalize({ hasConfirmedMeta, isPrivate }); +} + describe('context graph catch-up readiness classification', () => { const readinessBeforeCatchup = { version: 0,