diff --git a/package.json b/package.json index 3ace5061c3..a98953f588 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "build": "node scripts/build.mjs", "build:packages": "turbo build", - "build:runtime:packages": "pnpm -r --filter @origintrail-official/dkg-core --filter @origintrail-official/dkg-storage --filter @origintrail-official/dkg-query --filter @origintrail-official/dkg-publisher --filter @origintrail-official/dkg-chain --filter @origintrail-official/dkg-epcis --filter @origintrail-official/dkg-okf --filter @origintrail-official/dkg-random-sampling --filter @origintrail-official/dkg-agent --filter @origintrail-official/dkg-graph-viz --filter @origintrail-official/dkg-node-ui --filter @origintrail-official/dkg-adapter-openclaw --filter @origintrail-official/dkg-adapter-hermes --filter @origintrail-official/kafka-plugin --filter @origintrail-official/dkg run build", + "build:runtime:packages": "pnpm -r --filter @origintrail-official/dkg-core... --filter @origintrail-official/dkg-storage... --filter @origintrail-official/dkg-query... --filter @origintrail-official/dkg-publisher... --filter @origintrail-official/dkg-chain... --filter @origintrail-official/dkg-epcis... --filter @origintrail-official/dkg-okf... --filter @origintrail-official/dkg-random-sampling... --filter @origintrail-official/dkg-agent... --filter @origintrail-official/dkg-graph-viz... --filter @origintrail-official/dkg-node-ui... --filter @origintrail-official/dkg-adapter-openclaw... --filter @origintrail-official/dkg-adapter-hermes... --filter @origintrail-official/kafka-plugin... --filter @origintrail-official/dkg... run build", "build:runtime": "pnpm run build:runtime:packages && pnpm --filter @origintrail-official/dkg-node-ui run build:ui", "test": "turbo test && pnpm run test:scripts", "test:scripts": "node --test scripts/lib/__tests__/*.test.mjs", diff --git a/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py b/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py index 703c276bbd..25b972aaaa 100644 --- a/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py +++ b/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py @@ -47,6 +47,11 @@ ("a\rb", '"a\\rb"'), ("a\fb", '"a\\fb"'), ("a\bb", '"a\\bb"'), + # remaining ASCII controls -> UCHAR escapes (#416) + ("nul\u0000x", '"nul\\u0000x"'), + ("vt\u000bx", '"vt\\u000Bx"'), + ("us\u001fx", '"us\\u001Fx"'), + ("del\u007fx", '"del\\u007Fx"'), ] diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index 7fc76ac7d9..5dc4836716 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -891,9 +891,8 @@ export class DKGAgentBase { protected hostModeReconcilerTimer: ReturnType | null = null; protected hostModePruneTimer: ReturnType | null = null; // rc.9 PR-10: joinApprovalRetryQueue + joinApprovalRetryTimer - // deleted. The substrate's SQLite-backed ProtocolOutbox + its tick - // (`Messenger.processOutboxTick`) + opportunistic on-connect flush - // (`Messenger.processOutboxOnConnect`) replace the entire in-memory + // deleted. The substrate's SQLite-backed ProtocolOutbox + its + // scheduled tick (`Messenger.processOutboxTick`) replace the in-memory // queue: persistence across restart, generic per-protocol coverage, // identical backoff-ladder semantics. Operator-facing diagnostics // (`listPendingJoinApprovalRetries`) are stubbed to [] until PR-12 diff --git a/packages/agent/src/dkg-agent-constants.ts b/packages/agent/src/dkg-agent-constants.ts index 1744d36d05..d19eeba51b 100644 --- a/packages/agent/src/dkg-agent-constants.ts +++ b/packages/agent/src/dkg-agent-constants.ts @@ -172,27 +172,18 @@ export const STORAGE_ACK_REGISTRATION_RETRY_MS = 30_000; * the local curator state is correct but the invitee never learns to * sync, and their own retries can't help because they don't yet hold the * delegation that would let private-sync auth succeed. The tick walks the - * queue's `due()` entries with exponential backoff. Opportunistic retries - * also fire from `connection:open` when the invitee's peer reconnects, - * which usually wins the race; the timer is the safety net for cases - * where reconnect events are missed (e.g. relayed reconnects that don't - * surface a fresh `connection:open` on the curator). + * queue's `due()` entries with exponential backoff. This separate join queue + * retains its own peer lifecycle policy; Universal Messenger rows below are + * scheduled-only. */ export const JOIN_APPROVAL_RETRY_TICK_MS = 30_000; /** * Tick interval for the chat outbox retry queue. Same 30s cadence as * the join-approval queue (`JOIN_APPROVAL_RETRY_TICK_MS`). The cadence - * doesn't gate the FIRST retry — a backoff-due entry that's been - * waiting since 5s after first failure may sit idle for up to 25s - * before this tick picks it up — but the dominant retry trigger in - * practice is the `connection:open` opportunistic flush - * (`processMessageOutboxOnConnect`), which fires the moment the - * recipient peer becomes reachable again. The tick is the safety net - * for cases where reconnect events are missed (e.g. relayed reconnects - * that don't surface a fresh `connection:open` on the sender) or - * where the recipient was reachable all along but transport failures - * are coming from somewhere upstream of libp2p. + * is the sole automatic trigger: reconnect churn must not bypass a row's + * persisted `nextAttemptAt`. A due entry may sit for up to one tick interval + * before the scheduler picks it up. */ export const MESSAGE_OUTBOX_TICK_MS = 30_000; diff --git a/packages/agent/src/dkg-agent-join.ts b/packages/agent/src/dkg-agent-join.ts index 4d81fc168c..eedc4630b2 100644 --- a/packages/agent/src/dkg-agent-join.ts +++ b/packages/agent/src/dkg-agent-join.ts @@ -716,8 +716,7 @@ export class JoinRequestMethods extends DKGAgentBase { ctx, `join-approval for "${contextGraphId}" → ${agentAddress} not delivered now ` + `(error=${result.error ?? 'unknown'}). Curator-local state is correct; ` + - `substrate outbox holds the queued send and will retry on its backoff ` + - `ladder + on the invitee's next reconnect.`, + `substrate outbox holds the queued send and will retry on its backoff ladder.`, ); } @@ -727,8 +726,8 @@ export class JoinRequestMethods extends DKGAgentBase { * delivery state matters. * * Used by: - * * The substrate's periodic outbox tick + on-connect flush — - * both transparent to this call (rc.9 PR-10). + * * The substrate's periodic outbox tick, transparent to this call + * (rc.9 PR-10). * * The operator-facing route `POST /api/context-graph/{id}/redeliver-approval`, * which lets an operator (or peer agent via the chat MCP) re-poke * the curator when the automated retry isn't fast enough. @@ -856,17 +855,16 @@ export class JoinRequestMethods extends DKGAgentBase { */ // rc.9 PR-10: processJoinApprovalRetryQueueTick + // processJoinApprovalRetryQueueOnConnect deleted. The substrate's - // Messenger.processOutboxTick + Messenger.processOutboxOnConnect - // cover /dkg/10.0.1/join-request automatically (same as chat in + // Messenger.processOutboxTick covers /dkg/10.0.1/join-request + // automatically (same as chat in // PR-3), so the two dedicated processors are obsolete. Operator // re-fire route POST /api/context-graph/{id}/redeliver-approval is // unchanged — it still calls redeliverJoinApproval which now // simply re-issues the substrate send. /** - * Re-attempt delivery of a single chat outbox entry. Centralised so - * the periodic tick + the connection:open opportunistic flush share - * one code path. Returns the entry's current state so the caller can + * Re-attempt delivery of a single chat outbox entry from the periodic + * scheduler. Returns the entry's current state so the caller can * decide what to log. * * Goes through `messageHandler.sendChat` directly (bypassing @@ -884,17 +882,14 @@ export class JoinRequestMethods extends DKGAgentBase { * class: an inbound circuit connection from P was open and live, but * every `libp2p.dialProtocol(P, ...)` retry on our side failed with * "The dial request has no valid addresses for peer" for several - * minutes. Daemon logs showed 31 `connection:open` events from P - * (all inbound, all via R) + 20 opportunistic-flush attempts, all - * failing dialProtocol — and then the moment ONE outbound connection - * succeeded (which populated peerStore from outbound identify), the - * very next opportunistic-flush delivered the queued message. + * minutes. Reverse-path enrichment ensures the next scheduled retry sees a + * usable address without letting connection churn bypass persisted backoff. * * The clean fix would be inside libp2p (`dialProtocol` should reuse * an existing open connection of any direction — see PR 5 in the * postmortem follow-up plan), but until that lands, populating * peerStore from the inbound circuit's address gives the dialer - * something to find on the very next attempt. + * something to find on the next scheduled attempt. * * Public so a unit test can exercise it directly without standing up * a full libp2p network (the listener that calls it is registered @@ -1140,7 +1135,7 @@ export class JoinRequestMethods extends DKGAgentBase { ctx, `${label} for "${contextGraphId}" to ${agentAddress} (${targetPeerId}) ` + `queued in substrate outbox: ${sendResult.error}. ` + - `Substrate will retry on its own backoff ladder + on the invitee's next reconnect.`, + `Substrate will retry on its persisted backoff schedule.`, ); return { delivered: false, peerId: targetPeerId, error: sendResult.error }; } diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index ee3dd57f7d..5a74e4693c 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -228,7 +228,8 @@ import { resolveSyncResponderSnapshotBudgetOptions, } from './sync/responder/sync-handler.js'; import { runSyncOnConnect, SyncOnConnectPostSyncError, type SyncOnConnectOutcome, type SyncOnConnectPeerOutcome } from './sync/on-connect/sync-on-connect.js'; -import { mapWithConcurrency, CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/map-with-concurrency.js'; +import { mapWithConcurrency } from './map-with-concurrency.js'; +import { CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/catchup-concurrency.js'; import { getSyncBackpressureSnapshot, getSyncBackpressureBusyError, @@ -2345,14 +2346,8 @@ export class LifecycleSyncMethods extends DKGAgentBase { this.node.libp2p.addEventListener('connection:open', (evt) => { const remotePeer = evt.detail.remotePeer.toString(); if (remotePeer === this.node.libp2p.peerId.toString()) return; - // rc.9 PR-10: the dedicated join-approval on-connect flush is - // gone. The substrate's `Messenger.processOutboxOnConnect` (a - // few lines further down in this handler) now covers join- - // approved retries too, since /dkg/10.0.1/join-request is now - // a substrate-managed protocol. - // Reverse-path peerStore enrichment for inbound circuit-relay - // connections, then the symmetric chat-outbox flush. + // connections. // // Closes the "Window D" class from the May 2026 Miles↔Lex 6h // soak postmortem: an inbound circuit connection from peer P @@ -2365,17 +2360,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { // dialProtocol find an address and try it. // // User review on PR #536 caught the original ordering bug: - // running enrichment and the outbox flush in parallel - // fire-and-forget meant the first flush attempt could - // still hit `dialProtocol` against an EMPTY peerStore and - // fail with the same "no valid addresses" error this PR is - // meant to heal — pushing recovery onto the next 30s tick - // or another reconnect. Sequence the two: await enrichment - // first, then flush. Both stay wrapped in their own - // try/catch so an enrichment failure logs a warning and - // still lets the outbox flush proceed (it might succeed - // anyway via a stale-but-usable cached path). - // // The whole chain runs as a fire-and-forget IIFE so the // listener itself doesn't await — libp2p's // `connection:open` emitter is synchronous and we don't @@ -2396,17 +2380,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { const message = err instanceof Error ? err.message : String(err); this.log.warn(ctx, `Reverse-path peerStore enrichment failed for ${remotePeer}: ${message}`); } - // Universal Messenger substrate (rc.9 PR-2/PR-3): drain - // the generic outbox for this peer. Replaces the rc.8 - // chat-specific outbox flush — the substrate now carries - // chat (PR-3) and will carry every other short-message - // protocol after PR-8..PR-11. - try { - await this.messenger.processOutboxOnConnect(remotePeer); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - this.log.warn(ctx, `Opportunistic Messenger-outbox retry on connect failed for ${remotePeer}: ${message}`); - } // PR-2 (SWM-fanout plan): drain pending sender-key packages // that were queued because the recipient had no advertised // peerId at publish time. Tolerant of profile-lookup failure @@ -2604,8 +2577,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // MESSAGE_OUTBOX_TICK_MS for the rationale (silent-drop on // transport failure used to lose operator-typed messages from // `dkg_send_message`; this is the safety-net retry loop that turns - // them into eventual successes, complemented by the - // opportunistic-on-reconnect path in the connection:open listener). + // them into eventual successes on their persisted retry schedule. // Universal Messenger substrate retry tick (rc.9 PR-2 + // PR-3). The rc.8 chat-specific tick was deleted in PR-3; // this is now the only outbox tick — chat (PR-3) and every diff --git a/packages/agent/src/dkg-agent-publish.ts b/packages/agent/src/dkg-agent-publish.ts index 1ccb0ece7f..2ca33f5e04 100644 --- a/packages/agent/src/dkg-agent-publish.ts +++ b/packages/agent/src/dkg-agent-publish.ts @@ -101,7 +101,7 @@ import { assertQuadLiteralsMutf8Safe, } from '@origintrail-official/dkg-core'; import { SpanStatusCode } from '@opentelemetry/api'; -import { GraphManager, PrivateContentStore, createTripleStore, loadSelectedSharedMemoryQuads, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, createTripleStore, loadSharedMemoryQuadsForScope, resolveSharedMemoryScopeWriteGraph, type SharedMemoryGraphScope, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; import { EVMChainAdapter, NoChainAdapter, enrichEvmError, buildKnowledgeAssetUal, type EVMAdapterConfig, type ChainAdapter, type CreateContextGraphParams, type CreateOnChainContextGraphParams, type CreateOnChainContextGraphResult, type TxResult, type V10PublishingConvictionAccountInfo } from '@origintrail-official/dkg-chain'; import { DKGPublisher, PublishHandler, SharedMemoryHandler, UpdateHandler, ChainEventPoller, AccessHandler, AccessClient, @@ -146,6 +146,7 @@ import { type QueryRequest, type QueryResponse, type QueryAccessConfig, type LookupType, } from '@origintrail-official/dkg-query'; import { DKGAgentWallet, type AgentWallet } from './agent-wallet.js'; +import { unpackKnowledgeAssetId } from './ka-identity.js'; import { ProfileManager } from './profile-manager.js'; import { DiscoveryClient, type SkillSearchOptions, type DiscoveredAgent, type DiscoveredOffering } from './discovery.js'; @@ -426,6 +427,28 @@ function normalizeOptionalContextGraphId(value: string | null | undefined): stri return trimmed ? trimmed : undefined; } +function sharedMemoryScopeForFinalizedLifecycle( + authorAddress: string, + packedKaId: bigint | undefined, +): SharedMemoryGraphScope { + if (packedKaId === undefined) return { kind: 'complete-family' }; + const unpacked = unpackKnowledgeAssetId(packedKaId); + const sealedAuthor = ethers.getAddress(authorAddress); + const packedAuthor = BigInt(unpacked.agentAddress); + // Legacy/mock seals may carry only the low 96-bit KA number. Preserve that + // compatibility by binding a zero packed namespace to the sealed author; + // a real nonzero namespace must still match exactly. + if (packedAuthor !== 0n && ethers.getAddress(unpacked.agentAddress) !== sealedAuthor) { + throw new Error( + `Finalized lifecycle KA id ${packedKaId} is not in author ${sealedAuthor}'s namespace`, + ); + } + return { + kind: 'named-lifecycle', + identity: { agentAddress: sealedAuthor, kaNumber: unpacked.kaNumber }, + }; +} + function rejectOversizedRdfLiterals(quads: Quad[] | undefined, label: string): void { if (!quads || quads.length === 0) return; assertQuadLiteralsMutf8Safe(quads, { label }); @@ -3317,18 +3340,20 @@ export class PublishMethods extends DKGAgentBase { contextGraphId: string, selection: 'all' | { rootEntities: string[] }, subGraphName?: string, + scope: SharedMemoryGraphScope = { kind: 'complete-family' }, ): Promise { const swmGraph = contextGraphSharedMemoryUri(contextGraphId, subGraphName); - return loadSelectedSharedMemoryQuads(this.store, swmGraph, selection, { + const options = { querySource: 'agent.resolveLiftWorkspaceSlice', - rootEntitiesErrorMessage: ({ inputCount, hadInput }) => ( + rootEntitiesErrorMessage: ({ inputCount, hadInput }: { inputCount: number; hadInput: boolean }) => ( hadInput ? `_loadSelectedSWMQuads: no valid rootEntities provided ` + `(all ${inputCount} entries failed IRI validation) ` + `for context graph ${contextGraphId}` : `_loadSelectedSWMQuads: no rootEntities supplied for context graph ${contextGraphId}` ), - }); + } as const; + return loadSharedMemoryQuadsForScope(this.store, swmGraph, selection, scope, options); } /** @@ -3805,6 +3830,10 @@ export class PublishMethods extends DKGAgentBase { ); } } + const sharedMemoryScope = sharedMemoryScopeForFinalizedLifecycle( + seal.authorAddress, + seal.reservedKaId ?? packedKaId, + ); const newMerkleHexBare = ethers.hexlify(seal.merkleRoot).slice(2); let result: PublishResult; @@ -3815,6 +3844,7 @@ export class PublishMethods extends DKGAgentBase { [...request.roots], request.subGraphName, ctx, + sharedMemoryScope, ); } catch (err) { this.log.warn( @@ -4264,6 +4294,10 @@ export class PublishMethods extends DKGAgentBase { ); } } + const sharedMemoryScope = sharedMemoryScopeForFinalizedLifecycle( + seal.authorAddress, + seal.reservedKaId ?? packedKaId, + ); const newMerkleHexBare = ethers.hexlify(seal.merkleRoot).slice(2); @@ -4279,6 +4313,7 @@ export class PublishMethods extends DKGAgentBase { contextGraphId, { rootEntities: seal.rootEntities }, opts?.subGraphName, + sharedMemoryScope, ); const updateAttestation = await this._buildPrecomputedUpdateAttestationForSeal( packedKaId, @@ -4311,6 +4346,7 @@ export class PublishMethods extends DKGAgentBase { seal.rootEntities, opts?.subGraphName, opts?.operationCtx ?? createOperationContext('publishFromSWM'), + sharedMemoryScope, ); } catch (err) { this.log.warn( @@ -4397,6 +4433,7 @@ export class PublishMethods extends DKGAgentBase { contextGraphId, { rootEntities: seal.rootEntities }, opts?.subGraphName, + sharedMemoryScope, ); if (sealedSwmQuads.length === 0) { throw new Error( @@ -4412,8 +4449,8 @@ export class PublishMethods extends DKGAgentBase { subGraphName: opts?.subGraphName, publisherNodeIdentityIdOverride: opts?.publisherNodeIdentityIdOverride, publishEpochs: opts?.publishEpochs, - clearSharedMemoryAfter: opts?.clearSharedMemoryAfter, reservedKaId: recoveredReservedKaId, + sharedMemoryScope, // Wired through to the inner publisher.publish() via // publishFromSharedMemory's `precomputedAttestation` option. // Skips the publisher's signing entirely. @@ -4453,6 +4490,17 @@ export class PublishMethods extends DKGAgentBase { } } + // Exact scope owns published-root cleanup. A caller's explicit request to + // clear every remaining share is a separate family-wide destructive action + // that runs only after a confirmed publish/update. + if (result.status === 'confirmed' && opts?.clearSharedMemoryAfter === true) { + await publisher.clearRemainingSharedMemory( + contextGraphId, + opts?.subGraphName, + opts?.operationCtx ?? createOperationContext('publishFromSWM'), + ); + } + // OT-RFC-43 A2 (decision 2) — stamp the VM pointer on the lifecycle URN // whenever the publish/update is confirmed. (For the mint path this is the // first VM pointer; for the update path the DELETE/INSERT above already set @@ -4769,17 +4817,30 @@ export class PublishMethods extends DKGAgentBase { * CG-DID catalog subject is appended so it is in scope for BOTH the author seal * (`_loadSelectedSWMQuads`) and the publisher's reload — which scope identically. * For `selection: 'all'` the selection is returned unchanged (both already read - * the whole SWM graph). + * the whole SWM graph). The generated floor is written into the same explicit + * graph scope as the publish; otherwise an exact named-lifecycle read would + * correctly exclude a floor left in the legacy bucket. */ async _ensureCuratedCatalogInSwm(this: DKGAgent, contextGraphId: string, selection: 'all' | { rootEntities: string[] }, subGraphName: string | undefined, ctx: OperationContext, + scope: SharedMemoryGraphScope = { kind: 'complete-family' }, ): Promise<'all' | { rootEntities: string[] }> { const swmGraph = contextGraphSharedMemoryUri(contextGraphId, subGraphName); + const catalogTargetGraph = await resolveSharedMemoryScopeWriteGraph( + this.store, + swmGraph, + scope, + { source: 'agent.ensureCuratedCatalogInSwm' }, + ); const cgDid = contextGraphDataUri(contextGraphId); - const catalogQuads = buildPublicProjection({ ual: cgDid, accessPolicy: 'private', graph: swmGraph }); + const catalogQuads = buildPublicProjection({ + ual: cgDid, + accessPolicy: 'private', + graph: catalogTargetGraph, + }); await this.store.insert(catalogQuads); this.log.info( ctx, @@ -4844,6 +4905,7 @@ export class PublishMethods extends DKGAgentBase { * publisher then keeps its existing allocate-at-publish behavior. */ reservedKaId?: bigint; + sharedMemoryScope?: SharedMemoryGraphScope; /** * RFC-001 §9.x — pre-computed attestation captured by * `agent.assertion.finalize()`. When the caller has already @@ -4919,7 +4981,13 @@ export class PublishMethods extends DKGAgentBase { ? generatedPrivateCatalogTripleKeys(contextGraphId) : undefined; if (hasGeneratedPrivateCatalog) { - selection = await this._ensureCuratedCatalogInSwm(contextGraphId, selection, options?.subGraphName, ctx); + selection = await this._ensureCuratedCatalogInSwm( + contextGraphId, + selection, + options?.subGraphName, + ctx, + options?.sharedMemoryScope, + ); } // RFC-001 §9.x — selection-based publish bridge. If the caller @@ -4940,6 +5008,7 @@ export class PublishMethods extends DKGAgentBase { contextGraphId, selection, options?.subGraphName, + options?.sharedMemoryScope, ); if (swmQuads.length > 0) { resolvedSeal = await this._buildPrecomputedAttestationForSelection( @@ -5019,6 +5088,7 @@ export class PublishMethods extends DKGAgentBase { precomputedAttestation: resolvedSeal, // OT-RFC-43 A2 — reuse the finalize-stamped packed kaId (no re-allocate). reservedKaId: options?.reservedKaId, + sharedMemoryScope: options?.sharedMemoryScope, encryptInlinePayload, encryptInlineChunked, }); diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 59ed216dc6..0ec0003f03 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -618,7 +618,7 @@ function normalizeStorageAckConfig(config: DKGAgentConfig): ResolvedDKGAgentConf } interface ACKReliableMessenger { - sendReliable( + sendRequestOwned( peerId: string, protocol: string, data: Uint8Array, @@ -631,11 +631,11 @@ function createACKSendP2P(input: { timeoutMs: number; }): ACKCollectorDeps['sendP2P'] { return async (peerId: string, protocol: string, data: Uint8Array) => { - const sendResult = await input.messenger.sendReliable(peerId, protocol, data, { + const sendResult = await input.messenger.sendRequestOwned(peerId, protocol, data, { timeoutMs: input.timeoutMs, }); if (!sendResult.delivered) { - throw new Error(`substrate queued (transport): ${sendResult.error}`); + throw new Error(`substrate send already in flight (transport): ${sendResult.error}`); } if (!sendResult.response) { throw new Error('substrate delivered (transport) without response'); @@ -1548,6 +1548,14 @@ export class DKGAgent extends DKGAgentBase { clearInterval(this.messengerOutboxTimer); this.messengerOutboxTimer = null; } + try { + await this.messenger.stopOutboxDrain(); + } catch (error) { + this.log.warn( + createOperationContext('system'), + `DKGAgent.stop: outbox retry drain failed during shutdown: ${error instanceof Error ? error.message : String(error)}`, + ); + } if (this.swmAckQuorumTimer) { clearInterval(this.swmAckQuorumTimer); this.swmAckQuorumTimer = null; diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index f190393ed9..6450cea38e 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -228,7 +228,8 @@ export { // (`/api/context-graph/subscribe` → `catchup-runner-worker-impl`) runs the same // registry-scale per-peer fan-out and must be bounded by the SAME knob, without // deep-importing the compiled `dist/` module. -export { mapWithConcurrency, CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/map-with-concurrency.js'; +export { mapWithConcurrency } from './map-with-concurrency.js'; +export { CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/catchup-concurrency.js'; // 2026-07-08 sync-storm mitigation (#1233) — resolve the opt-in `agents/_meta` // fetch flag. Exported on the public surface so the CLI daemon lifecycle resolves // it identically to the in-agent lifecycle, without deep-importing `dist/`. diff --git a/packages/agent/src/map-with-concurrency.ts b/packages/agent/src/map-with-concurrency.ts new file mode 100644 index 0000000000..c8a334f690 --- /dev/null +++ b/packages/agent/src/map-with-concurrency.ts @@ -0,0 +1,29 @@ +// Bounded-concurrency ordered map shared by agent subsystems. Keeping this +// neutral avoids coupling p2p retry scheduling to the sync directory. + +/** + * Like `Promise.all(items.map(fn))` but with at most `limit` callbacks in + * flight. Results preserve input order. A rejecting callback rejects the call. + */ +export async function mapWithConcurrency( + items: readonly T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + if (items.length === 0) return []; + if (!Number.isInteger(limit) || limit <= 0 || limit >= items.length) { + return Promise.all(items.map((item, i) => fn(item, i))); + } + + const results = new Array(items.length); + let nextIndex = 0; + const worker = async (): Promise => { + for (;;) { + const i = nextIndex++; + if (i >= items.length) return; + results[i] = await fn(items[i], i); + } + }; + await Promise.all(Array.from({ length: limit }, () => worker())); + return results; +} diff --git a/packages/agent/src/p2p/messenger.ts b/packages/agent/src/p2p/messenger.ts index 812c3e9e4f..3434af1ded 100644 --- a/packages/agent/src/p2p/messenger.ts +++ b/packages/agent/src/p2p/messenger.ts @@ -11,6 +11,15 @@ import { type ProtocolOutboxEntry, type ProtocolRouter, } from '@origintrail-official/dkg-core'; +import { + OutboxDrainer, + type OutboxDrainerOptions, +} from './outbox-drainer.js'; +export { + DEFAULT_OUTBOX_DRAIN_BATCH_SIZE, + DEFAULT_OUTBOX_DRAIN_CONCURRENCY, + type OutboxDrainerOptions, +} from './outbox-drainer.js'; /** Bytes payload the substrate uses to signal `RESPONSE_GONE` on the wire. */ const RESPONSE_GONE_BYTES = new TextEncoder().encode(RESPONSE_GONE_MARKER); @@ -158,6 +167,11 @@ export interface MessengerDeps { * production uses the default `Date.now`. */ clock?: () => number; + /** + * Periodic retry scheduler bounds. Defaults and validation are owned by + * `OutboxDrainer` (`batchSize: 100`, `concurrency: 4`). + */ + outboxDrain?: OutboxDrainerOptions; } export interface SendOpts { @@ -250,6 +264,12 @@ export type ReliableSendResult = error: string; }; +/** The explicit throw policy can never produce a durable queued result. */ +export type ThrowingReliableSendResult = Exclude< + ReliableSendResult, + { delivered: false; queued: true } +>; + /** Handler signature for `Messenger.register`. */ export type ReliableHandler = ( payload: Uint8Array, @@ -324,13 +344,13 @@ export interface SloProtocolStats { * via `sloWindowSamples` in `MessengerDeps`. */ export const DEFAULT_SLO_WINDOW_SAMPLES = 1000; - export class Messenger { private readonly router: ProtocolRouter; private readonly idempotencyStore?: MessageIdempotencyStore; private readonly outbox?: ProtocolOutbox; private readonly clock: () => number; private readonly resolvePeer?: (peerId: string, opts: { signal: AbortSignal }) => Promise; + private readonly outboxDrainer?: OutboxDrainer; /** * Application handlers registered via `register`. Stored separately @@ -414,6 +434,13 @@ export class Messenger { this.clock = deps.clock ?? (() => Date.now()); this.sloWindowSamples = deps.sloWindowSamples ?? DEFAULT_SLO_WINDOW_SAMPLES; this.resolvePeer = deps.resolvePeer; + if (this.outbox) { + this.outboxDrainer = new OutboxDrainer( + (now, limit) => this.outbox!.duePage(now, limit), + (entry) => this.retryOutboxEntry(entry), + deps.outboxDrain, + ); + } } /** @@ -574,6 +601,26 @@ export class Messenger { protocolId: string, payload: Uint8Array, opts: SendReliableOpts = {}, + ): Promise { + return this.sendFramed(peerId, protocolId, payload, opts, true); + } + + /** Reliable framing/idempotency for bounded request-owned retries; never queues. */ + async sendRequestOwned( + peerId: string, + protocolId: string, + payload: Uint8Array, + opts: SendReliableOpts = {}, + ): Promise { + return this.sendFramed(peerId, protocolId, payload, opts, false) as Promise; + } + + private async sendFramed( + peerId: string, + protocolId: string, + payload: Uint8Array, + opts: SendReliableOpts, + queueRecoverableFailure: boolean, ): Promise { this.requireSubstrate('sendReliable'); @@ -633,8 +680,8 @@ export class Messenger { // Inflight guard (rc.9 #521 lesson lifted): two parallel // attempters on the same `(peer, protocol, messageId)` can race - // when the periodic tick + an opportunistic-flush fire close - // together. Second attempter exits without dialing. + // when a first sender + periodic retry (or overlapping explicit callers) + // race. Second attempter exits without dialing. if (!outbox.tryBeginAttempt(peerId, protocolId, messageId)) { // Another attempt is in flight. This is not the same thing as // durable queued: the winning attempt may still be on its first @@ -669,6 +716,13 @@ export class Messenger { if (!isRecoverableMessengerSendError(err, errMsg)) { throw err; } + if (!queueRecoverableFailure) { + // No durable row can later deliver or expire this request, so its SLO + // start marker has no future owner. Clear it before returning control + // to the request-scoped retry loop. + this.firstAttemptAt.delete(sloK); + throw err; + } const entry = outbox.enqueueFailure( peerId, protocolId, @@ -790,30 +844,18 @@ export class Messenger { * `dropExpired(now)` evicts it on age — recovering an encoding * bug requires operator action (manual replay or shutdown). */ - async processOutboxTick(now: number): Promise { - if (!this.outbox) return; - const due = this.outbox.due(now); - for (const entry of due) { - await this.retryOutboxEntry(entry); - } + processOutboxTick(now: number): Promise { + return this.outboxDrainer?.tick(now) ?? Promise.resolve(); } - /** - * Opportunistic-flush retry loop. The lifecycle.ts wiring (PR-3) - * calls this from a libp2p `connection:open` event for `peerId`: - * a reconnection is the signal we were waiting for, so attempt - * every pending entry for `peer` NOW even if `nextAttemptAt` is - * still in the future. - * - * Same guards as `processOutboxTick` — must check `hasEntry` - * after `tryBeginAttempt` to defend against the rc.9 #538 race. - */ - async processOutboxOnConnect(peerId: string): Promise { - if (!this.outbox) return; - const pending = this.outbox.pendingFor(peerId); - for (const entry of pending) { - await this.retryOutboxEntry(entry); - } + /** Await the currently active periodic drain during graceful shutdown. */ + async waitForOutboxDrain(): Promise { + await this.outboxDrainer?.wait(); + } + + /** Cancel the remainder of the loaded page and join already-started retries. */ + async stopOutboxDrain(): Promise { + await this.outboxDrainer?.stop(); } private async retryOutboxEntry(entry: { @@ -827,7 +869,7 @@ export class Messenger { return; } try { - // Stale-snapshot guard — between the moment `due`/`pendingFor` + // Stale-snapshot guard — between the moment `due` // gave us the snapshot and the moment `tryBeginAttempt` won, // a sibling flush may have completed delivery and called // `markDelivered`. Re-check `hasEntry` and bail if gone. @@ -852,20 +894,20 @@ export class Messenger { this.clearDhtWalkRateLimitIfDrained(entry.peer); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); + const updated = outbox.enqueueFailure( + entry.peer, + entry.protocol, + entry.messageId, + entry.payload, + errMsg, + this.clock(), + ); if (isRecoverableMessengerSendError(err, errMsg)) { - const updated = outbox.enqueueFailure( - entry.peer, - entry.protocol, - entry.messageId, - entry.payload, - errMsg, - this.clock(), - ); this.maybeScheduleDhtWalk(entry.peer, updated.attempts, errMsg); } - // Non-recoverable: leave the entry alone. `dropExpired` will - // age it out; an operator-facing diagnostic surface (PR-12) - // will surface stuck entries so a human can intervene. + // A non-recoverable retry remains visible for operator intervention, but + // advances on the backoff ladder so it cannot permanently occupy the + // front of every bounded due page and starve later deliverable rows. } finally { outbox.endAttempt(entry.peer, entry.protocol, entry.messageId); } @@ -879,7 +921,7 @@ export class Messenger { * * Fire-and-forget: never blocks the caller. The DHT walk's * side-effect (populating `peerStore` for the peer) heals the - * next opportunistic-flush or periodic-tick retry, not the + * next periodic-tick retry, not the * current one. This is intentional — the current retry has * already failed; the walk is for the next attempt. * @@ -923,7 +965,7 @@ export class Messenger { } private clearDhtWalkRateLimitIfDrained(peerId: string): void { - if (!this.outbox || this.outbox.pendingFor(peerId).length === 0) { + if (!this.outbox || !this.outbox.hasPendingFor(peerId)) { this.lastDhtWalkAt.delete(peerId); } } diff --git a/packages/agent/src/p2p/outbox-drainer.ts b/packages/agent/src/p2p/outbox-drainer.ts new file mode 100644 index 0000000000..7c2fba386c --- /dev/null +++ b/packages/agent/src/p2p/outbox-drainer.ts @@ -0,0 +1,93 @@ +import { mapWithConcurrency } from '../map-with-concurrency.js'; + +export const DEFAULT_OUTBOX_DRAIN_BATCH_SIZE = 100; +export const DEFAULT_OUTBOX_DRAIN_CONCURRENCY = 4; + +export interface OutboxDrainerOptions { + batchSize?: number; + concurrency?: number; +} + +interface ResolvedOutboxDrainerOptions { + batchSize: number; + concurrency: number; +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback; + if (!Number.isInteger(resolved) || resolved <= 0) { + throw new RangeError(`OutboxDrainer ${name} must be a positive integer`); + } + return resolved; +} + +/** Shutdown-safe bounded scheduler: its active promise covers every started worker. */ +export class OutboxDrainer { + private active: Promise | null = null; + private stopping = false; + private readonly options: ResolvedOutboxDrainerOptions; + + constructor( + private readonly loadDue: (now: number, limit: number) => readonly T[], + private readonly processEntry: (entry: T) => Promise, + options: OutboxDrainerOptions = {}, + ) { + this.options = { + batchSize: positiveInteger( + options.batchSize, + DEFAULT_OUTBOX_DRAIN_BATCH_SIZE, + 'batchSize', + ), + concurrency: positiveInteger( + options.concurrency, + DEFAULT_OUTBOX_DRAIN_CONCURRENCY, + 'concurrency', + ), + }; + } + + tick(now: number): Promise { + if (this.stopping) return Promise.resolve(); + if (this.active) return this.active; + const drain = this.drain(now); + this.active = drain; + const clearActive = (): void => { + if (this.active === drain) this.active = null; + }; + void drain.then(clearActive, clearActive); + return drain; + } + + async wait(): Promise { + await this.active; + } + + /** Stop admitting work and join retries that had already started. */ + async stop(): Promise { + this.stopping = true; + await this.active; + } + + private async drain(now: number): Promise { + const due = this.loadDue(now, this.options.batchSize).slice(0, this.options.batchSize); + const results = await mapWithConcurrency( + due, + this.options.concurrency, + async (entry): Promise<{ failed: true; error: unknown } | undefined> => { + if (this.stopping) return undefined; + try { + await this.processEntry(entry); + return undefined; + } catch (error) { + return { failed: true, error }; + } + }, + ); + const failures = results + .filter((result): result is { failed: true; error: unknown } => result !== undefined) + .map((result) => result.error); + if (failures.length > 0) { + throw new AggregateError(failures, `${failures.length} outbox retry worker(s) failed`); + } + } +} diff --git a/packages/agent/src/sync/catchup-concurrency.ts b/packages/agent/src/sync/catchup-concurrency.ts new file mode 100644 index 0000000000..83c31c0ba6 --- /dev/null +++ b/packages/agent/src/sync/catchup-concurrency.ts @@ -0,0 +1,5 @@ +/** Sync-owned catch-up policy; generic worker-pool mechanics live separately. */ +export const CATCHUP_MAX_CONCURRENT_PEER_SYNCS: number = (() => { + const raw = Number(process.env.DKG_CATCHUP_MAX_CONCURRENT_PEERS); + return Number.isInteger(raw) && raw > 0 ? raw : 4; +})(); diff --git a/packages/agent/src/sync/map-with-concurrency.ts b/packages/agent/src/sync/map-with-concurrency.ts deleted file mode 100644 index 6f6d834952..0000000000 --- a/packages/agent/src/sync/map-with-concurrency.ts +++ /dev/null @@ -1,57 +0,0 @@ -// map-with-concurrency.ts -// -// Bounded-concurrency ordered map. The catch-up fan-out -// (`runCatchupOverPeers`) previously fired a full durable+SWM sync at EVERY -// connected sync-capable peer via one unbounded `Promise.all` — the top -// amplifier of the 2026-07-07 mainnet "sync storm": one `/api/subscribe` on a -// large-degree node launched N concurrent full-CG pulls, saturating the triple -// store and the muxer. This caps the number of peer syncs running AT ONCE -// without dropping any peer (coverage preserved, just staggered) and without -// changing the result shape callers aggregate over — the returned array is in -// input order, one entry per item, exactly like `Promise.all(items.map(fn))`. - -/** - * Like `Promise.all(items.map(fn))` but with at most `limit` callbacks - * in flight at any moment. Results preserve input order. `fn` receives the - * item and its original index. A rejecting `fn` rejects the whole call (same - * as `Promise.all`) — callers that want per-item isolation must catch inside - * `fn`, as the catch-up fan-out already does. - * - * `limit <= 0` or `limit >= items.length` degrades to a plain `Promise.all` - * (no pool overhead, byte-identical behaviour to the pre-cap code). - */ -export async function mapWithConcurrency( - items: readonly T[], - limit: number, - fn: (item: T, index: number) => Promise, -): Promise { - if (items.length === 0) return []; - if (!Number.isInteger(limit) || limit <= 0 || limit >= items.length) { - return Promise.all(items.map((item, i) => fn(item, i))); - } - - const results = new Array(items.length); - let nextIndex = 0; - const worker = async (): Promise => { - for (;;) { - const i = nextIndex++; - if (i >= items.length) return; - results[i] = await fn(items[i], i); - } - }; - // `limit` workers drain the shared cursor; each awaits its item before - // pulling the next, so no more than `limit` `fn` calls are ever in flight. - await Promise.all(Array.from({ length: limit }, () => worker())); - return results; -} - -/** - * Max peer syncs the catch-up fan-out runs concurrently. Kept small so a - * high-degree node's subscribe/reconcile round can't flood its own triple - * store; every selected peer is still synced, just in bounded waves. - * Env-overridable for operators who need to tune throughput vs. store load. - */ -export const CATCHUP_MAX_CONCURRENT_PEER_SYNCS: number = (() => { - const raw = Number(process.env.DKG_CATCHUP_MAX_CONCURRENT_PEERS); - return Number.isInteger(raw) && raw > 0 ? raw : 4; -})(); diff --git a/packages/agent/test/finalization-promote-extra.test.ts b/packages/agent/test/finalization-promote-extra.test.ts index 01d90b8184..acfcb1c20a 100644 --- a/packages/agent/test/finalization-promote-extra.test.ts +++ b/packages/agent/test/finalization-promote-extra.test.ts @@ -439,16 +439,15 @@ describe('A-4: e2e — agent.publish() data lands in canonical (data) view post- // GH #1264 promotion: a confirmed one-shot `publish()` now stores its public // data in BOTH the per-KA verifiable-memory graph (the publish write) AND the // scoped RS-prover graph `/context/` (promoteConfirmedKCToScopedGraph). - // The verifiable-memory view unions both (dkg-query-engine.ts re-includes the - // per-cgId data graphs for #1098), so the SAME triple is observed once per - // source graph — two rows. The invariant is unchanged: the confirmed publish - // is immediately observable via VM with the correct value. Assert that the two - // rows are exactly the two known sources AND carry only the published value, - // so a third copy (count) or a wrong-member/garbage row (distinct set) still fails. + // The verifiable-memory read-both includes both (dkg-query-engine.ts + // re-includes the per-cgId data graphs for #1098), but collapses an identical + // full solution mapping already produced by an earlier mirror graph. The + // canonical triple is therefore observed once without applying DISTINCT to + // the caller projection (which would also erase legitimate bag multiplicity). expect( vmQr.bindings.length, - 'VM view observes the confirmed publish in its two source graphs (per-KA VM graph + scoped #1264 promotion)', - ).toBe(2); + 'VM view deduplicates the identical per-KA VM + scoped #1264 mirror', + ).toBe(1); expect( new Set(vmQr.bindings.map((b) => b['o'])), 'every VM-view row must carry the published value (no wrong-member or stale row)', diff --git a/packages/agent/test/map-with-concurrency.test.ts b/packages/agent/test/map-with-concurrency.test.ts index 74ad79fb7a..ce27c112ac 100644 --- a/packages/agent/test/map-with-concurrency.test.ts +++ b/packages/agent/test/map-with-concurrency.test.ts @@ -5,7 +5,8 @@ // unchanged), and no more than `limit` callbacks are ever in flight (so a // high-degree node's subscribe round can't flood its own store). import { describe, it, expect } from 'vitest'; -import { mapWithConcurrency, CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from '../src/sync/map-with-concurrency.js'; +import { mapWithConcurrency } from '../src/map-with-concurrency.js'; +import { CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from '../src/sync/catchup-concurrency.js'; const tick = () => new Promise((r) => setTimeout(r, 0)); diff --git a/packages/agent/test/messenger-substrate.test.ts b/packages/agent/test/messenger-substrate.test.ts index c6c9574bda..5d92d90d97 100644 --- a/packages/agent/test/messenger-substrate.test.ts +++ b/packages/agent/test/messenger-substrate.test.ts @@ -9,7 +9,11 @@ import { type ProtocolRouter, type StreamHandler, } from '@origintrail-official/dkg-core'; -import { Messenger, MessengerNotConfiguredError } from '../src/p2p/messenger.js'; +import { + DEFAULT_OUTBOX_DRAIN_BATCH_SIZE, + Messenger, + MessengerNotConfiguredError, +} from '../src/p2p/messenger.js'; /** * Hand-rolled call recorder: records every call's args on `.calls` @@ -58,7 +62,9 @@ interface RouterDouble { inboundHandler?: StreamHandler; } -function makeRouter(sendImpl?: () => Promise): RouterDouble { +function makeRouter( + sendImpl?: (...args: [string, string, Uint8Array, ...unknown[]]) => Promise, +): RouterDouble { const send = recorder( (sendImpl ?? (async () => new Uint8Array([0x10]))) as ( ...args: [string, string, Uint8Array, ...unknown[]] @@ -143,6 +149,22 @@ describe('Messenger.sendReliable (happy path semantics)', () => { }); }); +describe('Messenger.sendRequestOwned', () => { + it('keeps reliable framing but does not persist recoverable failures', async () => { + const router = makeRouter(async () => { + throw new Error('no valid addresses for peer'); + }); + const { messenger, outboxStore } = makeSubstrate({ router }); + + await expect(messenger.sendRequestOwned(PEER_A, PROTO, new Uint8Array([1]), { + messageId: FIXED_MSG_ID, + })).rejects.toThrow('no valid addresses for peer'); + expect(outboxStore.size()).toBe(0); + expect(() => decodeReliableEnvelope(router.send.calls[0][2] as Uint8Array)).not.toThrow(); + expect((messenger as any).firstAttemptAt.size).toBe(0); + }); +}); + describe('Messenger.sendReliable (sender-side idempotency)', () => { it('returns the cached response on a second send with the same messageId, no router call', async () => { const router = makeRouter(async () => new Uint8Array([0x42])); @@ -395,6 +417,116 @@ describe('Messenger.processOutboxTick (retry loop semantics)', () => { await messenger.processOutboxTick(clock() + 100); expect(router.send.calls.length).toBe(sendCallsBefore); }); + + it('coalesces overlapping ticks and bounds batch size and retry concurrency', async () => { + let active = 0; + let maxActive = 0; + const releases: Array<() => void> = []; + const router = makeRouter(async () => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => releases.push(resolve)); + active -= 1; + return new Uint8Array([0x42]); + }); + const idempotencyStore = new InMemoryMessageIdempotencyStore(); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + for (let i = 0; i < 5; i += 1) { + outboxStore.enqueue(PEER_A, PROTO, `message-${i}`, new Uint8Array([i]), 'offline', 0); + } + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore, + outboxStore, + backoffs: [10], + outboxDrain: { batchSize: 3, concurrency: 2 }, + }); + + const first = messenger.processOutboxTick(100); + const overlapping = messenger.processOutboxTick(100); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(router.send.calls).toHaveLength(2); + expect(maxActive).toBe(2); + releases.splice(0).forEach((release) => release()); + await new Promise((resolve) => setTimeout(resolve, 0)); + releases.splice(0).forEach((release) => release()); + await Promise.all([first, overlapping]); + + expect(router.send.calls).toHaveLength(3); + expect(outboxStore.size()).toBe(2); + }); + + it('caps a production-default tick at the default batch size', async () => { + const router = makeRouter(async () => new Uint8Array([0x42])); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + const queued = DEFAULT_OUTBOX_DRAIN_BATCH_SIZE + 25; + for (let i = 0; i < queued; i += 1) { + outboxStore.enqueue(PEER_A, PROTO, `default-batch-${i}`, new Uint8Array([i]), 'offline', 0); + } + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore: new InMemoryMessageIdempotencyStore(), + outboxStore, + backoffs: [10], + }); + + await messenger.processOutboxTick(100); + + expect(router.send.calls).toHaveLength(DEFAULT_OUTBOX_DRAIN_BATCH_SIZE); + expect(outboxStore.size()).toBe(25); + }); + + it('moves terminal failures behind later due rows instead of starving the next page', async () => { + const router = makeRouter(async (_peer, _protocol, payload) => { + if (payload[0] < 2) throw new Error('Invalid payload'); + return new Uint8Array([0x42]); + }); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + for (let i = 0; i < 3; i += 1) { + outboxStore.enqueue(PEER_A, PROTO, `terminal-${i}`, new Uint8Array([i]), 'offline', 0); + } + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore: new InMemoryMessageIdempotencyStore(), + outboxStore, + backoffs: [10], + clock: () => 100, + outboxDrain: { batchSize: 2, concurrency: 1 }, + }); + + await messenger.processOutboxTick(100); + await messenger.processOutboxTick(100); + expect(router.send.calls).toHaveLength(3); + expect(outboxStore.size()).toBe(2); + }); + + it('waitForOutboxDrain stays pending until the active retry completes', async () => { + let release!: () => void; + const router = makeRouter(async () => { + await new Promise((resolve) => { release = resolve; }); + return new Uint8Array([0x42]); + }); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + outboxStore.enqueue(PEER_A, PROTO, FIXED_MSG_ID, new Uint8Array([1]), 'offline', 0); + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore: new InMemoryMessageIdempotencyStore(), + outboxStore, + backoffs: [10], + }); + + const tick = messenger.processOutboxTick(100); + let waitResolved = false; + const waiting = messenger.waitForOutboxDrain().then(() => { waitResolved = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(router.send.calls).toHaveLength(1); + expect(waitResolved).toBe(false); + + release(); + await Promise.all([tick, waiting]); + expect(waitResolved).toBe(true); + expect(outboxStore.size()).toBe(0); + }); }); describe('Messenger construction guardrails', () => { diff --git a/packages/agent/test/outbox-drainer.test.ts b/packages/agent/test/outbox-drainer.test.ts new file mode 100644 index 0000000000..b2a0d6e0a3 --- /dev/null +++ b/packages/agent/test/outbox-drainer.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import { OutboxDrainer } from '../src/p2p/outbox-drainer.js'; + +describe('OutboxDrainer', () => { + it('keeps wait pending until every started worker settles after a sibling failure', async () => { + let release!: () => void; + const blocked = new Promise((resolve) => { release = resolve; }); + const drainer = new OutboxDrainer( + () => ['blocked', 'failed'], + async (entry) => { + if (entry === 'failed') throw new Error('store write failed'); + await blocked; + }, + { batchSize: 2, concurrency: 2 }, + ); + + const tick = drainer.tick(100); + let waitSettled = false; + const waiting = drainer.wait().catch(() => {}).then(() => { waitSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(waitSettled).toBe(false); + release(); + await expect(tick).rejects.toThrow('outbox retry worker'); + await waiting; + expect(waitSettled).toBe(true); + }); + + it('defensively caps a due loader that ignores the requested limit', async () => { + const processed: number[] = []; + const drainer = new OutboxDrainer( + () => [1, 2, 3, 4], + async (entry) => { processed.push(entry); }, + { batchSize: 2, concurrency: 1 }, + ); + + await drainer.tick(100); + expect(processed).toEqual([1, 2]); + }); + + it('starts a fresh drain after a failed tick', async () => { + let fail = true; + let loads = 0; + const drainer = new OutboxDrainer( + () => { loads += 1; return ['entry']; }, + async () => { if (fail) throw new Error('store write failed'); }, + { batchSize: 1, concurrency: 1 }, + ); + + await expect(drainer.tick(100)).rejects.toThrow('outbox retry worker'); + fail = false; + await drainer.tick(200); + expect(loads).toBe(2); + }); + + it('stops pulling new entries while joining retries already in flight', async () => { + let release!: () => void; + const blocked = new Promise((resolve) => { release = resolve; }); + const started: number[] = []; + const drainer = new OutboxDrainer( + () => [1, 2, 3], + async (entry) => { started.push(entry); await blocked; }, + { batchSize: 3, concurrency: 1 }, + ); + + const tick = drainer.tick(100); + await new Promise((resolve) => setTimeout(resolve, 0)); + const stopping = drainer.stop(); + let stopSettled = false; + void stopping.then(() => { stopSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(started).toEqual([1]); + expect(stopSettled).toBe(false); + release(); + await stopping; + expect(stopSettled).toBe(true); + await tick; + expect(started).toEqual([1]); + }); + + it('rejects invalid scheduler bounds at its own boundary', () => { + expect(() => new OutboxDrainer(() => [], async () => {}, { batchSize: 0, concurrency: 1 })) + .toThrow('batchSize must be a positive integer'); + expect(() => new OutboxDrainer(() => [], async () => {}, { batchSize: 1, concurrency: 0 })) + .toThrow('concurrency must be a positive integer'); + }); +}); diff --git a/packages/agent/test/outbox-shutdown-lifecycle.test.ts b/packages/agent/test/outbox-shutdown-lifecycle.test.ts new file mode 100644 index 0000000000..fabc0e4897 --- /dev/null +++ b/packages/agent/test/outbox-shutdown-lifecycle.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from 'vitest'; +import { DKGAgent } from '../src/dkg-agent.js'; + +describe('DKGAgent outbox shutdown lifecycle', () => { + it('cancels and awaits the active outbox drain before network teardown', async () => { + let release!: () => void; + const activeDrain = new Promise((resolve) => { release = resolve; }); + const stopOutboxDrain = vi.fn(() => activeDrain); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + store: { close: vi.fn(async () => {}) }, + log: { warn: vi.fn() }, + }); + + const stopping = agent.stop(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(stopOutboxDrain).toHaveBeenCalledOnce(); + expect(stopNode).not.toHaveBeenCalled(); + + release(); + await stopping; + expect(stopNode).toHaveBeenCalledOnce(); + }); + + it('logs a failed outbox drain and continues network teardown', async () => { + const stopOutboxDrain = vi.fn(async () => { throw new Error('drain failed'); }); + const stopNode = vi.fn(async () => {}); + const closeStore = vi.fn(async () => {}); + const warn = vi.fn(); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + store: { close: closeStore }, + log: { warn }, + }); + + await expect(agent.stop()).resolves.toBeUndefined(); + + expect(stopOutboxDrain).toHaveBeenCalledOnce(); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('outbox retry drain failed during shutdown: drain failed'), + ); + }); +}); diff --git a/packages/agent/test/p2p-resilience.test.ts b/packages/agent/test/p2p-resilience.test.ts index 2976508b34..9499586d04 100644 --- a/packages/agent/test/p2p-resilience.test.ts +++ b/packages/agent/test/p2p-resilience.test.ts @@ -338,21 +338,7 @@ describe('p2p resilience hooks', () => { } }, 15_000); - // User review on PR #536: the `connection:open` listener used to - // call `enrichPeerStoreFromInboundCircuit` and the outbox flush - // in parallel fire-and-forget, which meant the first outbox - // flush attempt could still hit `dialProtocol` against an EMPTY - // peerStore — exact same "no valid addresses for peer" failure - // the PR is meant to heal. Fix wraps both in a sequential IIFE - // so the flush sees the freshly-merged reverse-path address. - // - // rc.9 PR-3 substrate cutover: the outbox flush moved from - // `processMessageOutboxOnConnect` (chat-specific, deleted) to - // `messenger.processOutboxOnConnect` (substrate, generic). The - // ordering invariant is the same — enrich BEFORE the substrate - // flush so the substrate's dialProtocol calls see the merged - // peerStore address. - it('awaits enrichment BEFORE the substrate outbox flush on inbound circuit open (PR #536 review, rc.9 substrate)', async () => { + it('enriches reverse paths without waking the Messenger outbox on connection open', async () => { const agent = await DKGAgent.create({ name: 'ReversePathEnrichBeforeFlush', listenHost: '127.0.0.1', @@ -363,8 +349,25 @@ describe('p2p resilience hooks', () => { allowAllNetworkAdmission(agent); const events: string[] = []; + const wireProtocols: string[] = []; let enrichResolve: (() => void) | null = null; const enrichDone = new Promise((r) => { enrichResolve = r; }); + const remotePeer = freshPeerIdString(); + const testProtocol = '/dkg/test/scheduled-only'; + const queuedAt = Date.now(); + const messenger = (agent as any).messenger; + messenger.outbox.enqueueFailure( + remotePeer, + testProtocol, + 'scheduled-only-message', + new Uint8Array([1, 2, 3]), + 'offline', + queuedAt, + ); + messenger.router.send = async (_peer: string, protocol: string) => { + wireProtocols.push(protocol); + return new Uint8Array([0x42]); + }; (agent as any).enrichPeerStoreFromInboundCircuit = async () => { events.push('enrich:start'); @@ -372,11 +375,6 @@ describe('p2p resilience hooks', () => { events.push('enrich:end'); enrichResolve?.(); }; - (agent as any).messenger.processOutboxOnConnect = async () => { - events.push('flush:start'); - }; - - const remotePeer = freshPeerIdString(); const relayPeer = freshPeerIdString(); agent.node.libp2p.dispatchEvent(new CustomEvent('connection:open', { detail: { @@ -391,23 +389,22 @@ describe('p2p resilience hooks', () => { } as any)); await enrichDone; - // Give the IIFE one more tick to schedule the flush. + // Give the connection lifecycle's remaining async work time to settle. await new Promise(r => setTimeout(r, 50)); expect(events).toContain('enrich:start'); expect(events).toContain('enrich:end'); - expect(events).toContain('flush:start'); - expect(events.indexOf('enrich:end')).toBeLessThan(events.indexOf('flush:start')); + expect(wireProtocols).not.toContain(testProtocol); + + // The same row remains deliverable by the sole automatic trigger once + // its persisted backoff has elapsed. + await messenger.processOutboxTick(queuedAt + 10_000); + expect(wireProtocols.filter((protocol) => protocol === testProtocol)).toHaveLength(1); + expect(messenger.outbox.hasPendingFor(remotePeer)).toBe(false); } finally { await agent.stop().catch(() => {}); } }, 15_000); - // Stale-snapshot guard regression: the equivalent test for the - // substrate's `Messenger.processOutboxOnConnect` lives in - // `messenger-substrate.test.ts` ("honours the stale-snapshot - // guard (rc.9 #538)") — same race, same fix, just at the - // generic substrate layer rather than the chat-specific one - // which was deleted in rc.9 PR-3. }); }); diff --git a/packages/agent/test/publish-finalized-agent-lane.test.ts b/packages/agent/test/publish-finalized-agent-lane.test.ts index 01b2b9428e..227c3ee643 100644 --- a/packages/agent/test/publish-finalized-agent-lane.test.ts +++ b/packages/agent/test/publish-finalized-agent-lane.test.ts @@ -2,9 +2,14 @@ import { describe, expect, it } from 'vitest'; import { buildAssertionSealQuads, contextGraphAssertionUri, + contextGraphDataUri, + contextGraphSharedMemoryUri, + assertionLifecycleUri, contextGraphMetaUri, + createOperationContext, } from '@origintrail-official/dkg-core'; import { OxigraphStore, type Quad } from '@origintrail-official/dkg-storage'; +import { KA_ID_PRED, VM_CURRENT_ASSERTION_PRED } from '@origintrail-official/dkg-publisher'; import { DKGAgent } from '../src/dkg-agent.js'; const CG = 'publish-agent-lane'; @@ -25,6 +30,39 @@ function makeLog() { } describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { + it('injects the curated catalog floor into the exact named lifecycle graph', async () => { + const store = new OxigraphStore(); + const agent = Object.create(DKGAgent.prototype) as any; + agent.store = store; + agent.log = makeLog(); + const swmGraph = contextGraphSharedMemoryUri(CG); + const exactGraph = `${swmGraph}/${AGENT_B}/1`; + const cgDid = contextGraphDataUri(CG); + + const selection = await agent._ensureCuratedCatalogInSwm( + CG, + { rootEntities: [ROOT] }, + undefined, + createOperationContext('test'), + { + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }, + ); + + expect(selection).toEqual({ rootEntities: [ROOT, cgDid] }); + const exact = await store.query( + `SELECT ?p ?o WHERE { GRAPH <${exactGraph}> { <${cgDid}> ?p ?o } }`, + ); + const legacyBucket = await store.query( + `SELECT ?p ?o WHERE { GRAPH <${swmGraph}> { <${cgDid}> ?p ?o } }`, + ); + expect(exact.type).toBe('bindings'); + expect(exact.type === 'bindings' ? exact.bindings : []).toHaveLength(4); + expect(legacyBucket.type).toBe('bindings'); + expect(legacyBucket.type === 'bindings' ? legacyBucket.bindings : []).toHaveLength(0); + }); + it('reads finalized assertions from the explicitly selected non-default agent lane', async () => { const store = new OxigraphStore(); const assertionUri = contextGraphAssertionUri(CG, AGENT_B, NAME); @@ -51,7 +89,16 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { subGraphName?: string; }> = []; const publishCalls: Array<{ contextGraphId: string; selection: any; opts: any }> = []; - const loadCalls: Array<{ contextGraphId: string; selection: any; subGraphName?: string }> = []; + const remainingClearCalls: any[][] = []; + const loadCalls: Array<{ + contextGraphId: string; + selection: any; + subGraphName?: string; + scope?: { kind: 'complete-family' } | { + kind: 'named-lifecycle'; + identity: { agentAddress: string; kaNumber: bigint }; + }; + }> = []; const agent = Object.create(DKGAgent.prototype) as any; agent.store = store; @@ -73,13 +120,18 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { return agentAddress === AGENT_B; }, clearSwmShareComplete: async () => {}, + clearRemainingSharedMemory: async (...args: any[]) => { remainingClearCalls.push(args); }, }; agent._loadSelectedSWMQuads = async ( contextGraphId: string, selection: any, subGraphName?: string, + scope?: { kind: 'complete-family' } | { + kind: 'named-lifecycle'; + identity: { agentAddress: string; kaNumber: bigint }; + }, ) => { - loadCalls.push({ contextGraphId, selection, subGraphName }); + loadCalls.push({ contextGraphId, selection, subGraphName, scope }); return [{ subject: ROOT, predicate: 'http://schema.org/name', @@ -94,7 +146,7 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { ual: 'did:dkg:test/31337/1', merkleRoot: MERKLE, kaManifest: [], - status: 'tentative', + status: 'confirmed', publicQuads: [], }; }; @@ -103,6 +155,7 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { const result = await agent.publishFromFinalizedAssertion(CG, NAME, { agentAddress: AGENT_B, + clearSharedMemoryAfter: true, }); expect(result.assertionUri).toBe(assertionUri); @@ -116,6 +169,10 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { contextGraphId: CG, selection: { rootEntities: [ROOT] }, subGraphName: undefined, + scope: { + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }, }]); expect(publishCalls).toHaveLength(1); expect(publishCalls[0]).toMatchObject({ @@ -124,6 +181,10 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { }); expect(publishCalls[0]?.opts).toMatchObject({ reservedKaId: RESERVED_KA_ID, + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }, precomputedAttestation: { expectedMerkleRoot: MERKLE, authorAddress: AGENT_B, @@ -131,6 +192,87 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { reservedKaId: RESERVED_KA_ID, }, }); - expect(result.status).toBe('tentative'); + expect(publishCalls[0]?.opts).not.toHaveProperty('clearSharedMemoryAfter'); + expect(remainingClearCalls).toHaveLength(1); + expect(remainingClearCalls[0].slice(0, 2)).toEqual([CG, undefined]); + expect(result.status).toBe('confirmed'); + }); + + it('cleans only the finalized named lifecycle after a confirmed update', async () => { + const store = new OxigraphStore(); + const assertionUri = contextGraphAssertionUri(CG, AGENT_B, NAME); + const metaGraph = contextGraphMetaUri(CG); + const lifecycleUri = assertionLifecycleUri(CG, AGENT_B, NAME); + await store.insert([ + ...buildAssertionSealQuads({ + assertionUri, + metaGraph, + merkleRoot: MERKLE, + authorAddress: AGENT_B, + authorAttestationR: new Uint8Array(32).fill(1), + authorAttestationVS: new Uint8Array(32).fill(2), + authorSchemeVersion: 1, + chainId: 31337n, + kav10Address: AGENT_B, + reservedKaId: RESERVED_KA_ID, + finalizedAtIso: '2026-01-01T00:00:00.000Z', + rootEntities: [ROOT], + }) as Quad[], + { subject: lifecycleUri, predicate: VM_CURRENT_ASSERTION_PRED, object: '"prior"', graph: metaGraph }, + { subject: lifecycleUri, predicate: KA_ID_PRED, object: '"1"', graph: metaGraph }, + ]); + + const cleanupCalls: any[][] = []; + const loadCalls: any[][] = []; + const agent = Object.create(DKGAgent.prototype) as any; + agent.store = store; + agent.chain = {}; + agent.defaultAgentAddress = AGENT_B; + Object.defineProperty(agent, 'peerId', { value: 'peer-update', configurable: true }); + agent.log = makeLog(); + agent.publisher = { + hasSwmShareComplete: async () => true, + clearSwmShareComplete: async () => {}, + clearPublishedSwmRoots: async (...args: any[]) => { cleanupCalls.push(args); }, + }; + agent._loadSelectedSWMQuads = async (...args: any[]) => { + loadCalls.push(args); + return [{ + subject: ROOT, + predicate: 'http://schema.org/name', + object: '"updated"', + graph: '', + }]; + }; + agent._buildPrecomputedUpdateAttestationForSeal = async () => ({ + expectedNewMerkleRoot: MERKLE, + authorAddress: AGENT_B, + signature: { r: new Uint8Array(32), vs: new Uint8Array(32) }, + schemeVersion: 1, + }); + agent.update = async () => ({ + kaId: RESERVED_KA_ID, + ual: 'did:dkg:test/update/1', + merkleRoot: MERKLE, + kaManifest: [], + status: 'confirmed', + publicQuads: [], + }); + + const result = await agent.publishFromFinalizedAssertion(CG, NAME, { agentAddress: AGENT_B }); + + expect(result.status).toBe('confirmed'); + expect(loadCalls).toHaveLength(1); + expect(loadCalls[0].slice(0, 3)).toEqual([CG, { rootEntities: [ROOT] }, undefined]); + expect(loadCalls[0][3]).toEqual({ + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }); + expect(cleanupCalls).toHaveLength(1); + expect(cleanupCalls[0].slice(0, 3)).toEqual([CG, [ROOT], undefined]); + expect(cleanupCalls[0][4]).toEqual({ + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }); }); }); diff --git a/packages/agent/test/v10-ack-provider-wiring.test.ts b/packages/agent/test/v10-ack-provider-wiring.test.ts index b89af6aa51..52d7ce549c 100644 --- a/packages/agent/test/v10-ack-provider-wiring.test.ts +++ b/packages/agent/test/v10-ack-provider-wiring.test.ts @@ -268,12 +268,12 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # agent = boot.agent; const internals = boot.internals; const response = new Uint8Array([9]); - const sendReliable = vi.fn(async () => ({ + const sendRequestOwned = vi.fn(async () => ({ delivered: true, response, })); const payload = new Uint8Array([1, 2, 3]); - internals.messenger = { sendReliable }; + internals.messenger = { sendRequestOwned }; internals.createV10ACKProvider('test-cg'); const publishDeps = capturedAckCollectorDeps[0] as ACKCollectorDepsCapture; @@ -283,10 +283,10 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # const updateDeps = capturedAckCollectorDeps[1] as ACKCollectorDepsCapture; await expect(updateDeps.sendP2P!('peer-b', '/dkg/test/storage-update-ack', payload)).resolves.toEqual(response); - expect(sendReliable).toHaveBeenNthCalledWith(1, 'peer-a', '/dkg/test/storage-ack', payload, { + expect(sendRequestOwned).toHaveBeenNthCalledWith(1, 'peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 60_000, }); - expect(sendReliable).toHaveBeenNthCalledWith(2, 'peer-b', '/dkg/test/storage-update-ack', payload, { + expect(sendRequestOwned).toHaveBeenNthCalledWith(2, 'peer-b', '/dkg/test/storage-update-ack', payload, { timeoutMs: 60_000, }); }); @@ -296,18 +296,18 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # agent = boot.agent; const internals = boot.internals; const response = new Uint8Array([9]); - const sendReliable = vi.fn(async () => ({ + const sendRequestOwned = vi.fn(async () => ({ delivered: true, response, })); - internals.messenger = { sendReliable }; + internals.messenger = { sendRequestOwned }; const payload = new Uint8Array([1]); internals.createV10ACKProvider('test-cg'); const publishDeps = capturedAckCollectorDeps[0] as ACKCollectorDepsCapture; await expect(publishDeps.sendP2P!('peer-a', '/dkg/test/storage-ack', payload)).resolves.toEqual(response); - expect(sendReliable).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { + expect(sendRequestOwned).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 60_000, }); }); @@ -340,11 +340,11 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # agent = boot.agent; const internals = boot.internals; const response = new Uint8Array([9]); - const sendReliable = vi.fn(async () => ({ + const sendRequestOwned = vi.fn(async () => ({ delivered: true, response, })); - internals.messenger = { sendReliable }; + internals.messenger = { sendRequestOwned }; const payload = new Uint8Array([1]); internals.createV10ACKProvider('test-cg'); @@ -355,7 +355,7 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # handlerDeadlineMs: 0, sendTimeoutMs: 20_000, }); - expect(sendReliable).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { + expect(sendRequestOwned).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 20_000, }); }); @@ -370,10 +370,10 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # delivered: false, error: 'queued', })); - internals.messenger = { sendReliable: queuedSend }; + internals.messenger = { sendRequestOwned: queuedSend }; await expect( internals.createACKTransportFactory()().sendP2P('peer-a', '/dkg/test/storage-ack', payload), - ).rejects.toThrow(/substrate queued \(transport\): queued/); + ).rejects.toThrow(/substrate send already in flight \(transport\): queued/); expect(queuedSend).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 60_000, }); @@ -381,7 +381,7 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # const missingResponseSend = vi.fn(async () => ({ delivered: true, })); - internals.messenger = { sendReliable: missingResponseSend }; + internals.messenger = { sendRequestOwned: missingResponseSend }; await expect( internals.createACKTransportFactory()().sendP2P('peer-a', '/dkg/test/storage-ack', payload), ).rejects.toThrow(/substrate delivered \(transport\) without response/); diff --git a/packages/chain/src/chain-rpc-transport-error.ts b/packages/chain/src/chain-rpc-transport-error.ts index c815ea5de1..1b0039ab17 100644 --- a/packages/chain/src/chain-rpc-transport-error.ts +++ b/packages/chain/src/chain-rpc-transport-error.ts @@ -61,7 +61,6 @@ export class ChainRpcTransportError extends Error { readonly rpcUrls?: readonly string[]; readonly txHash?: string; - constructor( code: ChainRpcTransportCode, message: string, diff --git a/packages/chain/src/evm-adapter-base.ts b/packages/chain/src/evm-adapter-base.ts index 0b064cf3c9..5162ac9443 100644 --- a/packages/chain/src/evm-adapter-base.ts +++ b/packages/chain/src/evm-adapter-base.ts @@ -2591,7 +2591,15 @@ export class EVMChainAdapterBase { // endpoint that has it; best-effort `0` only when EVERY endpoint lacks it and // no transport error occurred. Non-retryable / transport-exhaustion errors // propagate (never masked as a bogus `0`). Order-independent — see the helper. - const block = await this.readProviderRetryingNull('getBlock', (p) => p.getBlock(blockNumber)); + const block = await this.rpcFailover.read( + 'getBlock', + (p) => p.getBlock(blockNumber), + { + rpcUsageConsumer: 'getBlock', + isEmptyResult: (value) => value == null, + endpointSetRetry: 'all-throttled', + }, + ); return block?.timestamp != null ? Number(block.timestamp) : 0; } diff --git a/packages/chain/src/evm-adapter-rpc.ts b/packages/chain/src/evm-adapter-rpc.ts index 896143a852..277852f847 100644 --- a/packages/chain/src/evm-adapter-rpc.ts +++ b/packages/chain/src/evm-adapter-rpc.ts @@ -153,6 +153,14 @@ export function isRetryableRpcError(err: unknown): boolean { .test(msg); } +/** Canonical classifier for provider throttling, shared by failover policy. */ +export function isThrottleRpcError(err: unknown): boolean { + if (err instanceof Error) enrichEvmError(err); + const status = errorStatus(err); + const message = errorMessage(err).toLowerCase(); + return status === 429 || /\b429\b|too many requests|rate[ -]?limit|throttl/.test(message); +} + export function assertSuccessfulReceipt(receipt: ethers.TransactionReceipt, label: string): void { if (receipt.status !== 0) return; const err = new Error(`${label} tx ${receipt.hash} was mined but reverted (status=0)`); diff --git a/packages/chain/src/rpc-failover-client.ts b/packages/chain/src/rpc-failover-client.ts index c0297b6cf7..a74b6d0587 100644 --- a/packages/chain/src/rpc-failover-client.ts +++ b/packages/chain/src/rpc-failover-client.ts @@ -41,7 +41,7 @@ import { JsonRpcProvider, Wallet, Contract, ethers } from 'ethers'; import { withSpan, getMetrics } from '@origintrail-official/dkg-core'; -import { withTimeout, isRetryableRpcError, isKnownTransactionError } from './evm-adapter-rpc.js'; +import { withTimeout, isRetryableRpcError, isThrottleRpcError, isKnownTransactionError, sleep } from './evm-adapter-rpc.js'; import { errorCode, errorMessage } from './evm-adapter-errors.js'; import { noteRpcFailover, noteRpcExhaustion, notePreferredEndpoint, noteRpcServed, rpcHost } from './rpc-failover-log.js'; import { EndpointStickiness, type StickinessIntent } from './endpoint-stickiness.js'; @@ -117,6 +117,21 @@ export interface ReadOpts { * sentinel polluting stickiness or telemetry. */ isEmptyResult?: (value: unknown) => boolean; + /** Retry a complete endpoint pass only when every failure was a throttle. */ + endpointSetRetry?: 'all-throttled'; +} + +type ProviderSetExhaustionKind = 'all-throttled' | 'mixed'; + +/** Internal exhaustion detail used only while deciding whether to retry a pass. */ +class ProviderSetExhaustedError extends ChainRpcTransportError { + constructor( + message: string, + readonly exhaustionKind: ProviderSetExhaustionKind, + opts: { cause: unknown; rpcUrls: readonly string[] }, + ) { + super('RPC_ENDPOINTS_EXHAUSTED', message, opts); + } } /** @@ -142,6 +157,9 @@ export interface RpcFailoverClientOptions { validateEndpoint?: ValidateEndpointFn; /** Endpoint-stickiness configuration (Mechanism B). */ stickiness?: StickinessOptions; + /** Full-pool retries after every endpoint reports a transient throttle. */ + readThrottleRetries?: number; + readThrottleBackoffMs?: number; } export interface StickinessOptions { @@ -205,6 +223,8 @@ export class RpcFailoverClient { private readonly stickiness: EndpointStickiness; /** Optional per-endpoint transport preflight (from `options.validateEndpoint`). */ private readonly validateEndpoint?: ValidateEndpointFn; + private readonly readThrottleRetries: number; + private readonly readThrottleBackoffMs: number; constructor( private readonly getEndpoints: () => RpcEndpoint[], @@ -219,6 +239,8 @@ export class RpcFailoverClient { options?: RpcFailoverClientOptions, ) { this.validateEndpoint = options?.validateEndpoint; + this.readThrottleRetries = options?.readThrottleRetries ?? 2; + this.readThrottleBackoffMs = options?.readThrottleBackoffMs ?? 250; const stickiness = options?.stickiness; const isEnabled = stickiness?.isEnabled ?? (stickiness?.enabled !== undefined ? () => stickiness.enabled as boolean : () => true); @@ -288,7 +310,7 @@ export class RpcFailoverClient { fn: (provider: JsonRpcProvider) => Promise, opts?: ReadOpts, ): Promise { - const run = () => this.runAcrossProviders( + const runPass = () => this.runAcrossProviders( label, fn, opts?.isRetryable ?? isRetryableRpcError, @@ -296,6 +318,9 @@ export class RpcFailoverClient { opts?.skipPreferred ?? false, opts?.isEmptyResult, ); + const run = opts?.endpointSetRetry === 'all-throttled' + ? () => this.withThrottleRetries(runPass) + : runPass; return opts?.rpcUsageConsumer ? withRpcUsageConsumer(opts.rpcUsageConsumer, run) : run(); } @@ -666,6 +691,7 @@ export class RpcFailoverClient { const attempts = this.stickiness.attempts(canonical, intent); const capMs = resolveCapMs(policy, canonical.length); let lastRetryable: unknown; + let allEndpointsThrottled = true; let sawEmpty = false; let lastEmpty: T | undefined; for (let i = 0; i < attempts.length; i += 1) { @@ -691,6 +717,7 @@ export class RpcFailoverClient { // telemetry. Try the next endpoint; if EVERY endpoint is empty (and none // errored) the empty value itself is the honest answer. sawEmpty = true; + allEndpointsThrottled = false; lastEmpty = out; continue; } @@ -700,6 +727,7 @@ export class RpcFailoverClient { } catch (err) { if (!isRetryable(err)) throw err; lastRetryable = err; + if (!isThrottleRpcError(err)) allEndpointsThrottled = false; attempt.recordFailure(); // de-prefer a failed backend if (!isLast) { noteRpcFailover(label, endpoint.rpcUrl, err, attempts[i + 1].endpoint.rpcUrl); @@ -725,7 +753,7 @@ export class RpcFailoverClient { ? errorMessage(lastRetryable) : `${label} read failed on all configured RPC endpoints ` + `(${canonical.map((e) => rpcHost(e.rpcUrl)).join(', ')}): ${errorMessage(lastRetryable)}`; - throw new ChainRpcTransportError('RPC_ENDPOINTS_EXHAUSTED', message, { + throw new ProviderSetExhaustedError(message, allEndpointsThrottled ? 'all-throttled' : 'mixed', { cause: lastRetryable, rpcUrls: canonical.map((e) => e.rpcUrl), }); @@ -743,6 +771,22 @@ export class RpcFailoverClient { ); } + private async withThrottleRetries(run: () => Promise): Promise { + for (let retry = 0; ; retry += 1) { + try { + return await run(); + } catch (error) { + // A raw 429 from a caller-supplied non-retryable classifier must retain + // its no-failover/no-retry contract. Only retry a typed FULL-POOL + // exhaustion produced by runAcrossProviders. + if (!(error instanceof ProviderSetExhaustedError) + || error.exhaustionKind !== 'all-throttled' + || retry >= this.readThrottleRetries) throw error; + await sleep(this.readThrottleBackoffMs * (2 ** retry)); + } + } + } + /** * Rebind a CONTRACT to `runner` (a provider for a view read) for one * per-endpoint attempt, leaving the caller's boot-bound handle untouched. The diff --git a/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts b/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts index 328a4a5c0e..5281aca031 100644 --- a/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts +++ b/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts @@ -181,6 +181,22 @@ describe('getBlockTimestamp: a null (unimported) receipt block fails over instea await expect(a.getBlockTimestamp(123n)).rejects.toMatchObject({ code: 'RPC_ENDPOINTS_EXHAUSTED' }); }); + it('retries an all-429 receipt-block pass and returns the recovered timestamp', async () => { + const retryable429 = () => { const e: any = new Error('429 too many requests'); e.status = 429; return e; }; + const primary = recorder(async () => { throw retryable429(); }); + let backupAttempt = 0; + const backup = recorder(async () => { + backupAttempt += 1; + if (backupAttempt === 1) throw retryable429(); + return { timestamp: 42 }; + }); + const a = makeTwoEndpointAdapter({ getBlock: primary }, { getBlock: backup }); + + await expect(a.getBlockTimestamp(123n)).resolves.toBe(42); + expect(primary.calls).toHaveLength(2); + expect(backup.calls).toHaveLength(2); + }); + it('MIXED null + transport error PROPAGATES regardless of endpoint order (order-independent, round-6 🔴)', async () => { const retryable429 = () => { const e: any = new Error('429 too many requests'); e.status = 429; return e; }; // Order A: primary transport error, backup null. A transport failure occurred, diff --git a/packages/chain/test/readwithfailover-loop.unit.test.ts b/packages/chain/test/readwithfailover-loop.unit.test.ts index 569b674d03..17a504e1f5 100644 --- a/packages/chain/test/readwithfailover-loop.unit.test.ts +++ b/packages/chain/test/readwithfailover-loop.unit.test.ts @@ -28,7 +28,7 @@ */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { EVMChainAdapter, type EVMAdapterConfig } from '../src/evm-adapter.js'; -import { RpcFailoverClient, type SignPopulatedFn } from '../src/rpc-failover-client.js'; +import { RpcFailoverClient, type RpcFailoverClientOptions, type SignPopulatedFn } from '../src/rpc-failover-client.js'; import { isChainRpcTransportError } from '../src/chain-rpc-transport-error.js'; import { getRpcFailoverStats, _resetRpcFailoverStatsForTest } from '../src/rpc-failover-log.js'; import { RPC_READ_STALL_TIMEOUT_MS } from '../src/evm-adapter-constants.js'; @@ -79,11 +79,12 @@ const NEVER_SIGN: SignPopulatedFn = async () => { * is the exact shape the adapter constructs it with, minus the adapter — so a * read failover regression is caught without a god-object back-reference. */ -function makeClient(providers: unknown[], rpcUrls: string[], signPopulated: SignPopulatedFn = NEVER_SIGN): RpcFailoverClient { +function makeClient(providers: unknown[], rpcUrls: string[], signPopulated: SignPopulatedFn = NEVER_SIGN, options?: RpcFailoverClientOptions): RpcFailoverClient { return new RpcFailoverClient( () => providers.map((p, i) => ({ provider: p as any, rpcUrl: rpcUrls[i] })), signPopulated, () => 'evm:31337', + options, ); } @@ -104,7 +105,9 @@ describe('RpcFailoverClient.read — read-failover loop logic (bare-mock, #1336) it('exhausts ALL endpoints → ChainRpcTransportError RPC_ENDPOINTS_EXHAUSTED, one attempt each', async () => { const primary = { read: recorder(async () => { throw retryable429(); }) }; const backup = { read: recorder(async () => { throw retryable429(); }) }; - const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example']); + const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example'], NEVER_SIGN, { + readThrottleRetries: 0, + }); let thrown: any; try { await client.read('unit read', (p: any) => p.read()); } catch (e) { thrown = e; } @@ -117,6 +120,42 @@ describe('RpcFailoverClient.read — read-failover loop logic (bare-mock, #1336) expect(backup.read.calls).toHaveLength(1); }); + it('backs off and retries the full pool when every endpoint returns 429', async () => { + let round = 0; + const primary = { read: recorder(async () => { throw retryable429(); }) }; + const backup = { read: recorder(async () => { + round += 1; + if (round === 1) throw retryable429(); + return 'recovered'; + }) }; + const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example'], NEVER_SIGN, { + readThrottleRetries: 2, + readThrottleBackoffMs: 1, + }); + + await expect(client.read('getBlock', (provider: any) => provider.read(), { + endpointSetRetry: 'all-throttled', + })) + .resolves.toBe('recovered'); + expect(primary.read.calls).toHaveLength(2); + expect(backup.read.calls).toHaveLength(2); + }); + + it('does not retry a mixed timeout plus 429 endpoint exhaustion', async () => { + const primary = { read: recorder(async () => { const error: any = new Error('timed out'); error.code = 'TIMEOUT'; throw error; }) }; + const backup = { read: recorder(async () => { throw retryable429(); }) }; + const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example'], NEVER_SIGN, { + readThrottleRetries: 2, + readThrottleBackoffMs: 1, + }); + + await expect(client.read('getBlock', (provider: any) => provider.read(), { + endpointSetRetry: 'all-throttled', + })).rejects.toMatchObject({ code: 'RPC_ENDPOINTS_EXHAUSTED' }); + expect(primary.read.calls).toHaveLength(1); + expect(backup.read.calls).toHaveLength(1); + }); + it('single-RPC: a retryable failure still stamps RPC_ENDPOINTS_EXHAUSTED but keeps the original message verbatim', async () => { const only = { read: recorder(async () => { throw new Error('connect ECONNREFUSED 127.0.0.1:8545'); }) }; const client = makeClient([only], ['https://only.example']); diff --git a/packages/cli/package.json b/packages/cli/package.json index 44d892b593..8cf51512cb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -15,7 +15,7 @@ } }, "scripts": { - "prebuild": "pnpm --filter @origintrail-official/dkg-adapter-openclaw run build && pnpm --filter @origintrail-official/dkg-adapter-hermes run build && pnpm --filter @origintrail-official/dkg-mcp run build && pnpm --filter @origintrail-official/dkg-okf run build", + "prebuild": "pnpm -r --filter @origintrail-official/dkg-adapter-openclaw... --filter @origintrail-official/dkg-adapter-hermes... --filter @origintrail-official/dkg-mcp... --filter @origintrail-official/dkg-okf... run build", "build": "tsc && node ../../scripts/copy-cli-runtime-assets.mjs", "prepack": "node ../../scripts/copy-cli-runtime-assets.mjs", "benchmark:catchup-runner": "node scripts/catchup-runner-benchmark.cjs", diff --git a/packages/cli/src/daemon/auto-update.ts b/packages/cli/src/daemon/auto-update.ts index 433138563c..194574161a 100644 --- a/packages/cli/src/daemon/auto-update.ts +++ b/packages/cli/src/daemon/auto-update.ts @@ -1543,11 +1543,9 @@ export async function performNpmUpdate( * version to `~/.dkg/previous-version` so `dkg rollback` (Edge * branch) has a target to reinstall. * - * Tradeoffs accepted (per RFC §7.2): - * - Non-atomic: a mid-install crash leaves the global state - * half-updated. Recovery is `npm install -g` re-run. - * - Network-dependent rollback: requires the npm registry to - * have the previous version available. + * The npm mutation itself is not atomic, so completion is followed by an + * executable/version self-check. A failed check immediately reinstalls the + * recorded previous version before the daemon is allowed to restart. * * The function returns `'updated'` after the npm install completes; * the caller is responsible for stopping the running daemon so the @@ -1617,14 +1615,7 @@ async function _performNpmUpdateInnerEdge( log(`Auto-update (npm-edge): running '${installCmd}'…`); try { const installStart = Date.now(); - await execAsyncIo(installCmd, { - encoding: "utf-8", - timeout: 300_000, - // Allow npm's progress / warning output to surface in the daemon - // log — operators tailing the log get real-time feedback on slow - // installs. stderr → stdout merge mirrors how npm itself runs - // interactively. - }); + await installGlobalCliVersion(execAsyncIo, targetVersion); const installMs = Date.now() - installStart; log(`Auto-update (npm-edge): npm install completed in ${installMs}ms.`); } catch (installErr: any) { @@ -1640,6 +1631,27 @@ async function _performNpmUpdateInnerEdge( return "failed"; } + try { + const reported = await verifyGlobalDkgVersion(execAsyncIo, targetVersion, 'self-check'); + log(`Auto-update (npm-edge): self-check passed (${reported}).`); + } catch (verifyErr: any) { + log(`Auto-update (npm-edge): post-install self-check failed — ${verifyErr?.message ?? verifyErr}.`); + if (!currentVersion) { + log('Auto-update (npm-edge): previous version is unknown; automatic rollback is unavailable.'); + return 'failed'; + } + const rollbackCmd = `npm install -g ${CLI_NPM_PACKAGE}@${currentVersion}`; + log(`Auto-update (npm-edge): rolling back with '${rollbackCmd}'…`); + try { + await installGlobalCliVersion(execAsyncIo, currentVersion); + const reported = await verifyGlobalDkgVersion(execAsyncIo, currentVersion, 'rollback'); + log(`Auto-update (npm-edge): rollback restored ${reported}.`); + } catch (rollbackErr: any) { + log(`Auto-update (npm-edge): CRITICAL rollback failed — ${rollbackErr?.message ?? rollbackErr}.`); + } + return 'failed'; + } + log( `Auto-update (npm-edge): ${CLI_NPM_PACKAGE}@${targetVersion} installed. ` + "Stop the daemon to restart from the new entry point.", @@ -1647,6 +1659,36 @@ async function _performNpmUpdateInnerEdge( return "updated"; } +type EdgeExec = typeof _autoUpdateIo.exec; + +async function installGlobalCliVersion(execIo: EdgeExec, version: string): Promise { + await execIo(`npm install -g ${CLI_NPM_PACKAGE}@${version}`, { + encoding: 'utf-8', + timeout: 300_000, + }); +} + +function parseReportedDkgVersion(output: string): string | undefined { + return output.match(/(?:^|\s)v?(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)(?=\s|$)/)?.[1]; +} + +async function verifyGlobalDkgVersion( + execIo: EdgeExec, + expectedVersion: string, + context: 'self-check' | 'rollback', +): Promise { + const { stdout, stderr } = await execIo('dkg --version', { + encoding: 'utf-8', + timeout: 30_000, + }); + const reported = `${stdout ?? ''} ${stderr ?? ''}`.trim(); + const parsed = parseReportedDkgVersion(reported); + if (parsed !== expectedVersion) { + throw new Error(`${context} expected ${expectedVersion}, got ${parsed ?? (reported || 'empty version output')}`); + } + return reported; +} + export async function checkForUpdate( au: ResolvedAutoUpdateConfig, log: (msg: string) => void, diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index dbc6f99039..4297ac48c3 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -99,7 +99,7 @@ import { slotEntryPoint, CLI_NPM_PACKAGE, } from '../config.js'; -import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../publisher-runner.js'; +import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type AsyncPublisherAvailability, type PublisherRuntime } from '../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../catchup-runner.js'; import { loadTokens, httpAuthGuard, extractBearerToken } from '../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; @@ -365,6 +365,7 @@ export async function handleRequest( admission: AdmissionStatsView, emitMemoryGraphChanged?: (event: MemoryGraphChangedEvent) => void, emitNotification?: (event: NotificationSseEvent) => void, + publisherAvailability?: AsyncPublisherAvailability, ): Promise { const url = new URL(req.url ?? "/", `http://${req.headers.host}`); const path = url.pathname; @@ -381,6 +382,7 @@ export async function handleRequest( agent, publisherControl, publisherRuntime, + publisherAvailability, config, startedAt, dashDb, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 687f5ebf3b..6aad669619 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -148,7 +148,7 @@ import { import { resolveOtelSignals, resolveLogExporterMode, isUnknownLogExporter } from '../telemetry-config.js'; import { createDaemonLogSink } from './log-sink.js'; import { startRpcUsageTelemetry } from './rpc-usage-log.js'; -import { createPublicSnapshotStore, createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../publisher-runner.js'; +import { createPublicSnapshotStore, createPublisherControlFromStore, resolveAsyncPublisherAvailability, startPublisherRuntimeIfEnabled, type AsyncPublisherAvailability, type PublisherRuntime } from '../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../catchup-runner.js'; import { loadTokens, httpAuthGuard } from '../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; @@ -1717,6 +1717,11 @@ export async function runDaemonInner( }); let publisherRuntime: PublisherRuntime | null = null; + let publisherAvailability: AsyncPublisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime: null, + lifecycleReason: config.publisher?.enabled ? 'publisher_starting' : 'publisher_disabled', + }); // Holds the running async-promote worker lifecycle (PR #3 of the // async-promote-queue series). Initialised in `startPostApiPublishing` // after the API is up so a recoverOnStartup hiccup never blocks boot; @@ -1988,7 +1993,17 @@ export async function runDaemonInner( log, }); publisherRuntime = runtime; + publisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime, + ...(runtime ? {} : { lifecycleReason: 'no_publisher_wallets' as const }), + }); } catch (err: any) { + publisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime: null, + lifecycleReason: 'publisher_startup_failed', + }); log(`Async publisher startup failed: ${err?.message ?? String(err)}`); } })(); @@ -3369,6 +3384,7 @@ export async function runDaemonInner( admissionStats, emitMemoryGraphChanged, emitNotification, + publisherAvailability, ); } catch (err: any) { // Single top-level error→HTTP mapping (in http-utils.ts diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index b525b44fa3..8b37447315 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -16,7 +16,7 @@ import type { OperationTracker, } from '@origintrail-official/dkg-node-ui'; import type { DkgConfig, loadNetworkConfig } from '../../config.js'; -import type { createPublisherControlFromStore, PublisherRuntime } from '../../publisher-runner.js'; +import type { AsyncPublisherAvailability, createPublisherControlFromStore, PublisherRuntime } from '../../publisher-runner.js'; import type { ExtractionStatusRecord } from '../../extraction-status.js'; import type { FileStore } from '../../file-store.js'; import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; @@ -60,6 +60,8 @@ export interface RequestContext { agent: DKGAgent; publisherControl: ReturnType; publisherRuntime: PublisherRuntime | null; + /** Lifecycle-owned publisher state; optional for direct route embeddings/tests. */ + publisherAvailability?: AsyncPublisherAvailability; config: DkgConfig; startedAt: number; dashDb: DashboardDB; diff --git a/packages/cli/src/daemon/routes/epcis.ts b/packages/cli/src/daemon/routes/epcis.ts index 14d8ca9b35..32bb1978a9 100644 --- a/packages/cli/src/daemon/routes/epcis.ts +++ b/packages/cli/src/daemon/routes/epcis.ts @@ -103,7 +103,7 @@ import { slotEntryPoint, CLI_NPM_PACKAGE, } from '../../config.js'; -import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; +import { createPublisherControlFromStore, resolveAsyncPublisherAvailability, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; import { loadTokens, httpAuthGuard, extractBearerToken } from '../../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; @@ -490,16 +490,24 @@ export async function handleEpcisRoutes(ctx: RequestContext): Promise { // POST /api/epcis/capture { contextGraphId?, subGraphName?, epcisDocument, publishOptions? } if (req.method === "POST" && path === "/api/epcis/capture") { - if (!config.publisher?.enabled) { + const publisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime: publisherRuntime, + lifecycleAvailability: ctx.publisherAvailability, + }); + if (!publisherAvailability.available && publisherAvailability.reason === 'publisher_disabled') { return jsonResponse(res, 503, { error: "PublisherDisabled", message: "Async EPCIS capture requires publisher.enabled=true", }); } - if (!publisherRuntime || publisherRuntime.walletIds.length === 0) { + if (!publisherAvailability.available) { return jsonResponse(res, 503, { error: "PublisherUnavailable", message: "Async EPCIS capture requires the publisher runtime to be running with at least one configured publisher wallet", + reason: publisherAvailability.reason, + retryable: publisherAvailability.retryable, + operatorActionRequired: publisherAvailability.operatorActionRequired, }); } const body = await readBody(req); diff --git a/packages/cli/src/daemon/routes/knowledge-assets.ts b/packages/cli/src/daemon/routes/knowledge-assets.ts index 76cc9b379d..16bdc91a81 100644 --- a/packages/cli/src/daemon/routes/knowledge-assets.ts +++ b/packages/cli/src/daemon/routes/knowledge-assets.ts @@ -49,6 +49,7 @@ import { SMALL_BODY_BYTES, } from "../http-utils.js"; import { validatePreSignedAuthorAttestation } from "./memory.js"; +import { resolveAsyncPublisherAvailability } from "../../publisher-runner.js"; import { recordAssertionActivity, recordConvictionCostCovered } from "../activity-notification.js"; import { handleKaImportArtifactResolve, @@ -1347,6 +1348,20 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< // to 400 (parity with the legacy publish path). if (layer === "vm" && verb === "publish-async") { try { + const publisherAvailability = resolveAsyncPublisherAvailability({ + config: ctx.config, + runtime: ctx.publisherRuntime, + lifecycleAvailability: ctx.publisherAvailability, + }); + if (!publisherAvailability.available) { + return jsonResponse(res, 503, { + code: "async_publisher_unavailable", + error: "The asynchronous publisher cannot accept jobs on this node.", + reason: publisherAvailability.reason, + retryable: publisherAvailability.retryable, + operatorActionRequired: publisherAvailability.operatorActionRequired, + }); + } const opts = resolveStandaloneVmPublishOptions(ctx, parsed); if (opts === null) return; const publishOptions = opts; diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index a69d7c951a..adac992a45 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -24,6 +24,50 @@ export interface PublisherRuntimeWallet { readonly identityId: bigint; } +export type AsyncPublisherUnavailableReason = + | 'publisher_disabled' + | 'publisher_starting' + | 'no_publisher_wallets' + | 'publisher_startup_failed'; + +export type AsyncPublisherAvailability = + | { available: true } + | { + available: false; + reason: AsyncPublisherUnavailableReason; + retryable: boolean; + operatorActionRequired: boolean; + }; + +/** + * Canonical readiness boundary for every async-ingress route. Lifecycle may + * supply an explicit starting/failure state; otherwise the runtime/config + * shape is classified consistently for direct route tests and embedded users. + */ +export function resolveAsyncPublisherAvailability(args: { + config: DkgConfig; + runtime: PublisherRuntime | null; + lifecycleReason?: AsyncPublisherUnavailableReason; + lifecycleAvailability?: AsyncPublisherAvailability; +}): AsyncPublisherAvailability { + if (args.lifecycleAvailability) return args.lifecycleAvailability; + if (args.runtime?.walletIds.length) return { available: true }; + const reason = args.lifecycleReason + ?? (args.runtime + ? 'no_publisher_wallets' + : args.config.publisher?.enabled + ? 'publisher_startup_failed' + : 'publisher_disabled'); + return { + available: false, + reason, + // Only the in-progress state can recover from the same client retry without + // operator/config/daemon intervention. + retryable: reason === 'publisher_starting', + operatorActionRequired: reason !== 'publisher_starting', + }; +} + export interface PublisherInspector { readonly publisher: AsyncLiftPublisher; readonly stop: () => Promise; diff --git a/packages/cli/test/helpers/live-daemon.ts b/packages/cli/test/helpers/live-daemon.ts index b2ddf0846e..683b1ba5f9 100644 --- a/packages/cli/test/helpers/live-daemon.ts +++ b/packages/cli/test/helpers/live-daemon.ts @@ -53,6 +53,8 @@ function uniquePort(base: number): number { export interface StartDaemonOpts { authEnabled?: boolean; + /** Enable a real async publisher and seed its wallet file. */ + publisherEnabled?: boolean; /** Extra keys merged into config.json (e.g. preset contextGraphs). */ extraConfig?: Record; readyTimeoutMs?: number; @@ -89,6 +91,7 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise = { ...process.env, @@ -150,6 +160,30 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise l.trim()).find((l) => l.length > 0 && !l.startsWith('#')) ?? null; if (!daemon.token) throw new Error('auth enabled but no token written'); } + if (opts.publisherEnabled) { + // `/api/status` becomes ready before the zero-delay publisher startup task. + // Poll the real async-ingress gate so tests never race `publisher_starting`. + for (let i = 0; i < 60; i += 1) { + const res = await fetch( + `${daemon.base}/api/knowledge-assets/publisher-readiness/wm/probe/vm/publish-async`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(daemon.token ? { Authorization: `Bearer ${daemon.token}` } : {}), + }, + body: '{}', + }, + ); + const body = await res.json().catch(() => ({})) as { code?: string; reason?: string }; + if (body.code !== 'async_publisher_unavailable') break; + if (body.reason !== 'publisher_starting') { + throw new Error(`Async publisher failed readiness: ${body.reason ?? res.status}`); + } + await sleep(100); + if (i === 59) throw new Error('Async publisher did not become ready in time'); + } + } return daemon; } diff --git a/packages/cli/test/knowledge-assets-1116-share-errors.test.ts b/packages/cli/test/knowledge-assets-1116-share-errors.test.ts index 41965ec74a..55bf7e358d 100644 --- a/packages/cli/test/knowledge-assets-1116-share-errors.test.ts +++ b/packages/cli/test/knowledge-assets-1116-share-errors.test.ts @@ -62,6 +62,12 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { agentOverrides: Record = {}, routeOverrides: { requestToken?: string; requestAgentAddress?: string } = {}, publisherControl: Record = {}, + publisherRuntime: unknown = { + walletIds: ['0x1111111111111111111111111111111111111111'], + wallets: [{ address: '0x1111111111111111111111111111111111111111' }], + }, + config: Record = {}, + publisherAvailability?: unknown, ) { const agent = { async listContextGraphs() { @@ -90,8 +96,9 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { res, agent, publisherControl, - publisherRuntime: null, - config: {}, + publisherRuntime, + config, + publisherAvailability, startedAt: Date.now(), dashDb: { insertNotification: () => 1 }, opWallets: {}, @@ -382,6 +389,73 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { expect(enqueueCalls).toBe(0); }); + it('vm/publish-async rejects before persisting when no runtime can claim jobs', async () => { + let resolved = 0; + let enqueued = 0; + await startWith({}, { + resolveFinalizedAssertionVmPublishIntent: async () => { resolved += 1; }, + }, {}, { + enqueueKnowledgeAssetVmPublish: async () => { enqueued += 1; }, + }, null); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + code: 'async_publisher_unavailable', + reason: 'publisher_disabled', + retryable: false, + operatorActionRequired: true, + }); + expect(resolved).toBe(0); + expect(enqueued).toBe(0); + }); + + it('vm/publish-async treats an empty-wallet runtime as operator-actionable', async () => { + let enqueued = 0; + await startWith({}, {}, {}, { + enqueueKnowledgeAssetVmPublish: async () => { enqueued += 1; }, + }, { walletIds: [], wallets: [] }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + code: 'async_publisher_unavailable', + reason: 'no_publisher_wallets', + retryable: false, + operatorActionRequired: true, + }); + expect(enqueued).toBe(0); + }); + + it('vm/publish-async uses lifecycle no-wallet state when the runtime is null', async () => { + let enqueued = 0; + await startWith({}, {}, {}, { + enqueueKnowledgeAssetVmPublish: async () => { enqueued += 1; }, + }, null, { publisher: { enabled: true } }, { + available: false, + reason: 'no_publisher_wallets', + retryable: false, + operatorActionRequired: true, + }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + reason: 'no_publisher_wallets', retryable: false, operatorActionRequired: true, + }); + expect(enqueued).toBe(0); + }); + + it('vm/publish-async classifies an unknown startup failure as operator-actionable', async () => { + await startWith({}, {}, {}, {}, null, { publisher: { enabled: true } }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + reason: 'publisher_startup_failed', retryable: false, operatorActionRequired: true, + }); + }); + it('vm/publish-async rejects a missing real share snapshot before enqueue', async () => { const store = await createTripleStore({ backend: 'oxigraph' }); let enqueueCalls = 0; diff --git a/packages/cli/test/publisher-availability.test.ts b/packages/cli/test/publisher-availability.test.ts new file mode 100644 index 0000000000..6c549edb7d --- /dev/null +++ b/packages/cli/test/publisher-availability.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { resolveAsyncPublisherAvailability, type PublisherRuntime } from '../src/publisher-runner.js'; + +const runtime = (wallets: unknown[]): PublisherRuntime => ({ + wallets, + walletIds: wallets.map((_, index) => String(index)), +} as unknown as PublisherRuntime); + +describe('resolveAsyncPublisherAvailability', () => { + it('classifies permanent configuration states as operator-actionable', () => { + expect(resolveAsyncPublisherAvailability({ config: {}, runtime: null })).toMatchObject({ + available: false, reason: 'publisher_disabled', retryable: false, operatorActionRequired: true, + }); + expect(resolveAsyncPublisherAvailability({ + config: { publisher: { enabled: true } } as any, + runtime: runtime([]), + })).toMatchObject({ + available: false, reason: 'no_publisher_wallets', retryable: false, operatorActionRequired: true, + }); + expect(resolveAsyncPublisherAvailability({ + config: { publisher: { enabled: true } } as any, + runtime: null, + lifecycleReason: 'publisher_startup_failed', + })).toMatchObject({ + available: false, reason: 'publisher_startup_failed', retryable: false, operatorActionRequired: true, + }); + }); + + it('marks only an in-progress startup as retryable and a funded runtime as ready', () => { + expect(resolveAsyncPublisherAvailability({ + config: { publisher: { enabled: true } } as any, + runtime: null, + lifecycleReason: 'publisher_starting', + })).toMatchObject({ + available: false, reason: 'publisher_starting', retryable: true, operatorActionRequired: false, + }); + expect(resolveAsyncPublisherAvailability({ config: {}, runtime: runtime([{}]) })).toEqual({ available: true }); + }); +}); diff --git a/packages/cli/test/rfc-41-bundle-b.test.ts b/packages/cli/test/rfc-41-bundle-b.test.ts index 1e5df4ded1..e888b67a85 100644 --- a/packages/cli/test/rfc-41-bundle-b.test.ts +++ b/packages/cli/test/rfc-41-bundle-b.test.ts @@ -213,7 +213,10 @@ describe('performNpmUpdateEdge (Bundle B1b)', () => { execCalls = []; _autoUpdateIo.exec = ((cmd: string, opts?: any): Promise<{ stdout: string; stderr: string }> => { execCalls.push({ cmd, opts }); - return Promise.resolve({ stdout: '', stderr: '' }); + return Promise.resolve({ + stdout: cmd === 'dkg --version' ? 'dkg 10.0.0-rc.12' : '', + stderr: '', + }); }) as any; }); @@ -227,8 +230,9 @@ describe('performNpmUpdateEdge (Bundle B1b)', () => { expect(result).toBe('updated'); expect(readFileSync(join(dkgHome, 'previous-version'), 'utf-8')).toBe('10.0.0-rc.11'); - expect(execCalls).toHaveLength(1); + expect(execCalls).toHaveLength(2); expect(execCalls[0].cmd).toBe('npm install -g @origintrail-official/dkg@10.0.0-rc.12'); + expect(execCalls[1].cmd).toBe('dkg --version'); expect(log.calls.some((m) => m.includes('10.0.0-rc.11 → ~/.dkg/previous-version'))).toBe(true); expect(log.calls.some((m) => m.includes('install completed'))).toBe(true); }); @@ -261,6 +265,55 @@ describe('performNpmUpdateEdge (Bundle B1b)', () => { expect(readFileSync(join(dkgHome, 'previous-version'), 'utf-8')).toBe('10.0.0-rc.11'); }); + it('rolls back when the installed CLI cannot pass its self-check', async () => { + let versionChecks = 0; + _autoUpdateIo.exec = (async (cmd: string, opts?: any) => { + execCalls.push({ cmd, opts }); + if (cmd === 'dkg --version') { + versionChecks += 1; + if (versionChecks === 1) throw new Error('dkg: command not found'); + return { stdout: 'dkg 10.0.0-rc.11', stderr: '' }; + } + return { stdout: '', stderr: '' }; + }) as any; + + const log = makeLog(); + const result = await performNpmUpdateEdge('10.0.0-rc.12', '10.0.0-rc.11', log.fn); + expect(result).toBe('failed'); + expect(execCalls.map((call) => call.cmd)).toEqual([ + 'npm install -g @origintrail-official/dkg@10.0.0-rc.12', + 'dkg --version', + 'npm install -g @origintrail-official/dkg@10.0.0-rc.11', + 'dkg --version', + ]); + expect(log.calls.some((message) => message.includes('rollback restored'))).toBe(true); + }); + + it('rolls back on an exact-version mismatch, including semver prefix collisions', async () => { + let versionChecks = 0; + _autoUpdateIo.exec = (async (cmd: string, opts?: any) => { + execCalls.push({ cmd, opts }); + if (cmd === 'dkg --version') { + versionChecks += 1; + return versionChecks === 1 + ? { stdout: 'dkg 10.0.0-rc.12', stderr: '' } + : { stdout: 'dkg 10.0.0-rc.0', stderr: '' }; + } + return { stdout: '', stderr: '' }; + }) as any; + + const log = makeLog(); + const result = await performNpmUpdateEdge('10.0.0-rc.1', '10.0.0-rc.0', log.fn); + expect(result).toBe('failed'); + expect(execCalls.map((call) => call.cmd)).toEqual([ + 'npm install -g @origintrail-official/dkg@10.0.0-rc.1', + 'dkg --version', + 'npm install -g @origintrail-official/dkg@10.0.0-rc.0', + 'dkg --version', + ]); + expect(log.calls.some((message) => message.includes('expected 10.0.0-rc.1'))).toBe(true); + }); + it('surfaces a prefix-configuration advisory on EACCES', async () => { _autoUpdateIo.exec = (() => { const err: any = new Error("EACCES: permission denied, mkdir '/usr/local/lib/node_modules'"); diff --git a/packages/cli/test/source-worker-daemon-client.test.ts b/packages/cli/test/source-worker-daemon-client.test.ts index 16136bee5c..11e1776d44 100644 --- a/packages/cli/test/source-worker-daemon-client.test.ts +++ b/packages/cli/test/source-worker-daemon-client.test.ts @@ -17,7 +17,7 @@ describe('source worker daemon client (real daemon)', () => { let daemon: LiveDaemon; beforeAll(async () => { - daemon = await startLiveDaemon(); + daemon = await startLiveDaemon({ publisherEnabled: true }); const created = await postJson(daemon, '/api/context-graph/create', { id: CG, name: CG, accessPolicy: 0 }); expect(created.status, `CG create failed: ${JSON.stringify(created.body)}`).toBeLessThan(300); }, 120_000); diff --git a/packages/cli/test/source-worker-runner.test.ts b/packages/cli/test/source-worker-runner.test.ts index 827fc72d33..7b22834a88 100644 --- a/packages/cli/test/source-worker-runner.test.ts +++ b/packages/cli/test/source-worker-runner.test.ts @@ -27,7 +27,7 @@ describe('source worker runner (real daemon)', () => { beforeAll(async () => { console.log = () => undefined; - daemon = await startLiveDaemon(); + daemon = await startLiveDaemon({ publisherEnabled: true }); const created = await postJson(daemon, '/api/context-graph/create', { id: CG, name: CG, accessPolicy: 0 }); expect(created.status, `CG create failed: ${JSON.stringify(created.body)}`).toBeLessThan(300); const sg = await postJson(daemon, '/api/sub-graph/create', { contextGraphId: CG, subGraphName: 'sg-1' }); diff --git a/packages/core/package.json b/packages/core/package.json index fd155c745c..526b00bbb3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -25,6 +25,7 @@ "clean": "rm -rf dist tsconfig.tsbuildinfo" }, "dependencies": { + "@origintrail-official/dkg-rdf-utils": "workspace:*", "@libp2p/autonat": "^3.0.20", "@libp2p/bootstrap": "^12.0.22", "@libp2p/circuit-relay-v2": "^4.2.5", diff --git a/packages/core/src/messenger-types.ts b/packages/core/src/messenger-types.ts index bb82654b5e..b31925fa6c 100644 --- a/packages/core/src/messenger-types.ts +++ b/packages/core/src/messenger-types.ts @@ -197,29 +197,36 @@ export interface ProtocolOutboxStore { /** * Whether an entry exists for `(peer, protocol, messageId)`. Used - * by the stale-snapshot guard in `Messenger.processOutboxOnConnect` - * — between `tryBeginAttempt` (inflight lock) and the wire send, + * by the scheduled drain's stale-snapshot guard — between + * `tryBeginAttempt` (inflight lock) and the wire send, * a sibling flush may have already delivered + removed the entry, * and we must not double-send. The rc9 #538 fix lifted into the * generic substrate. */ hasEntry(peer: string, protocol: string, messageId: string): boolean; + /** Whether this peer still has any durable row (DHT recovery bookkeeping). */ + hasPendingFor(peer: string): boolean; + /** - * All entries for a specific peer, regardless of `nextAttemptAt`. - * Used by `processOutboxOnConnect`: a reconnection is the signal - * we were waiting for, so attempt now even if backoff isn't due - * yet. Sorted by `firstFailureAt` ascending for FIFO per-peer - * drain. + * All entries whose `nextAttemptAt <= now`. + * + * This remains the required public store contract for compatibility with + * existing/custom stores. `ProtocolOutbox` applies canonical ordering when + * it turns this snapshot into a bounded retry page. */ - pendingFor(peer: string): ProtocolOutboxEntry[]; + due(now: number): ProtocolOutboxEntry[]; /** - * All entries whose `nextAttemptAt <= now`. Used by the periodic - * tick to find what's due for retry, regardless of peer - * reachability. + * Optional storage-level fast path for an ordered bounded retry page. + * + * `ProtocolOutbox` only calls this with a normalized non-negative integer + * limit. Implementations that opt in MUST select the first `limit` rows in + * ascending `nextAttemptAt`, `firstFailureAt`, then + * `(peer, protocol, messageId)` order. Stores that only implement the legacy + * `due(now)` API remain fully supported through the wrapper fallback. */ - due(now: number): ProtocolOutboxEntry[]; + duePage?(now: number, limit: number): ProtocolOutboxEntry[]; /** * Drop entries whose `firstFailureAt` is older than the @@ -250,6 +257,7 @@ export interface ProtocolOutboxStore { getEntry(peer: string, protocol: string, messageId: string): ProtocolOutboxEntry | undefined; } + /** * Durable per-author KA-number allocator (OT-RFC-43 Option-1 * deterministic KA identity, B2 allocator core). diff --git a/packages/core/src/protocol-outbox.ts b/packages/core/src/protocol-outbox.ts index 92351b8db6..c683e240b3 100644 --- a/packages/core/src/protocol-outbox.ts +++ b/packages/core/src/protocol-outbox.ts @@ -89,6 +89,19 @@ function cloneOutboxEntry(entry: ProtocolOutboxEntry): ProtocolOutboxEntry { return { ...entry, payload: cloneBytes(entry.payload) }; } +function compareDueEntries(a: ProtocolOutboxEntry, b: ProtocolOutboxEntry): number { + return a.nextAttemptAt - b.nextAttemptAt + || a.firstFailureAt - b.firstFailureAt + || a.peer.localeCompare(b.peer) + || a.protocol.localeCompare(b.protocol) + || a.messageId.localeCompare(b.messageId); +} + +function normalizeDuePageLimit(limit: number | undefined): number | undefined { + if (limit === undefined || !Number.isFinite(limit)) return undefined; + return Math.max(0, Math.floor(limit)); +} + interface ProtocolOutboxStorePolicy extends ProtocolOutboxOptions { backoffFor: (attempts: number) => number; } @@ -102,17 +115,9 @@ export class ProtocolOutbox { private readonly backoffs: readonly number[]; /** * Per-key inflight set to prevent concurrent retry attempts for the - * same `(peer, protocol, messageId)`. Two trigger surfaces — the - * periodic tick (`Messenger.processOutboxTick`) and the - * opportunistic flush on `connection:open` - * (`Messenger.processOutboxOnConnect`) — can interleave: the tick - * starts the send for entry E, JS yields, `connection:open` fires, - * the on-connect handler reads `pendingFor(peer)` (entry E is still - * there — `markDelivered` hasn't fired yet because the in-flight - * send hasn't resolved), and would start a CONCURRENT second send - * for the same entry. Worst case both succeed and the receiver sees - * the same payload twice (receiver dedup absorbs it, but we waste - * a round-trip and amplify load). + * same `(peer, protocol, messageId)`. Overlapping scheduler callers or + * another explicit sender can otherwise interleave around a stale due + * snapshot and duplicate the same wire attempt. * * `tryBeginAttempt` is an atomic check-and-set: the second * concurrent attempter sees `false` and exits without dialing. @@ -201,18 +206,29 @@ export class ProtocolOutbox { return this.store.hasEntry(peer, protocol, messageId); } - /** Entries whose `nextAttemptAt <= now`. */ + /** All due entries in deterministic retry order. */ due(now: number): ProtocolOutboxEntry[] { - return this.store.due(now); + return this.duePage(now); } /** - * All entries for `peer`, regardless of `nextAttemptAt`. Used by - * `Messenger.processOutboxOnConnect` for opportunistic flush on - * reconnection. + * Return a canonical retry page while preserving legacy `due(now)` stores. + * Stores may opt into the bounded fast path; the fallback sorts before it + * caps so an older store cannot bypass either the order or the batch bound. */ - pendingFor(peer: string): ProtocolOutboxEntry[] { - return this.store.pendingFor(peer); + duePage(now: number, limit?: number): ProtocolOutboxEntry[] { + const normalizedLimit = normalizeDuePageLimit(limit); + if (normalizedLimit === 0) return []; + + const snapshot = normalizedLimit !== undefined && this.store.duePage + ? this.store.duePage(now, normalizedLimit) + : this.store.due(now); + const ordered = [...snapshot].sort(compareDueEntries); + return normalizedLimit === undefined ? ordered : ordered.slice(0, normalizedLimit); + } + + hasPendingFor(peer: string): boolean { + return this.store.hasPendingFor(peer); } /** Drop entries older than the store's configured max-age. */ @@ -327,19 +343,21 @@ export class InMemoryProtocolOutboxStore implements ProtocolOutboxStore { return this.entries.has(InMemoryProtocolOutboxStore.key(peer, protocol, messageId)); } - pendingFor(peer: string): ProtocolOutboxEntry[] { - return Array.from(this.entries.values()) - .filter((e) => e.peer === peer) - .sort((a, b) => a.firstFailureAt - b.firstFailureAt) - .map(cloneOutboxEntry); + hasPendingFor(peer: string): boolean { + return Array.from(this.entries.values()).some((entry) => entry.peer === peer); } due(now: number): ProtocolOutboxEntry[] { return Array.from(this.entries.values()) .filter((e) => e.nextAttemptAt <= now) + .sort(compareDueEntries) .map(cloneOutboxEntry); } + duePage(now: number, limit: number): ProtocolOutboxEntry[] { + return this.due(now).slice(0, limit); + } + dropExpired(now: number): ProtocolOutboxEntry[] { const dropped: ProtocolOutboxEntry[] = []; for (const [key, entry] of this.entries) { @@ -356,12 +374,12 @@ export class InMemoryProtocolOutboxStore implements ProtocolOutboxStore { } list(): ProtocolOutboxEntry[] { - return Array.from(this.entries.values()).map((e) => ({ ...e })); + return Array.from(this.entries.values()).map(cloneOutboxEntry); } getEntry(peer: string, protocol: string, messageId: string): ProtocolOutboxEntry | undefined { const entry = this.entries.get(InMemoryProtocolOutboxStore.key(peer, protocol, messageId)); - return entry ? { ...entry } : undefined; + return entry ? cloneOutboxEntry(entry) : undefined; } } diff --git a/packages/core/src/publisher-extension.ts b/packages/core/src/publisher-extension.ts index 69101216c9..a87560903d 100644 --- a/packages/core/src/publisher-extension.ts +++ b/packages/core/src/publisher-extension.ts @@ -1,3 +1,9 @@ +import { + escapeRdfLiteral, + isRdfTerm, + normalizeRdfObject, +} from '@origintrail-official/dkg-rdf-utils'; + export interface DkgPublisherExtensionQuadInput { subject: unknown; predicate: unknown; @@ -146,25 +152,14 @@ export function normalizeDkgPublisherQuads( })); } -// NOTE: the MCP adapter inlines a byte-for-byte copy of this normalizer (it is -// deliberately dep-light and does not import dkg-core) as `normalizeRdfObject` in -// `packages/mcp-dkg/src/tools/assertions.ts`. If you change the behavior here, -// update that copy AND its golden fixture in -// `packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts` so the -// public `dkg_knowledge_asset_create({quads})` object contract stays identical -// across the MCP / OpenClaw / Hermes adapters. +// Core and MCP preserve their public names as thin compatibility wrappers around +// the complete dependency-free normalizer in @origintrail-official/dkg-rdf-utils. export function normalizeDkgPublisherObject(value: unknown): string { - const raw = String(value ?? ''); - if (isDkgRdfTerm(raw)) return raw; - return `"${escapeDkgRdfLiteral(raw)}"`; + return normalizeRdfObject(value); } export function isDkgRdfTerm(value: string): boolean { - return ( - /^(?:https?:\/\/|urn:|did:)/i.test(value) || - value.startsWith('_:') || - value.startsWith('"') - ); + return isRdfTerm(value); } /** @@ -172,14 +167,7 @@ export function isDkgRdfTerm(value: string): boolean { * Returns only the escaped body; callers wrap it in quotes. */ export function escapeDkgRdfLiteral(value: string): string { - return value - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/\t/g, '\\t') - .replace(/\f/g, '\\f') - .replace(/\x08/g, '\\b'); + return escapeRdfLiteral(value); } export { diff --git a/packages/core/test/protocol-outbox.test.ts b/packages/core/test/protocol-outbox.test.ts index 110dbb193f..a85a28f8db 100644 --- a/packages/core/test/protocol-outbox.test.ts +++ b/packages/core/test/protocol-outbox.test.ts @@ -5,7 +5,11 @@ import { InMemoryProtocolOutboxStore, ProtocolOutbox, } from '../src/protocol-outbox.js'; -import { RESPONSE_CACHE_BYTES } from '../src/messenger-types.js'; +import { + RESPONSE_CACHE_BYTES, + type ProtocolOutboxEntry, + type ProtocolOutboxStore, +} from '../src/messenger-types.js'; const PEER_A = '12D3KooWMilesPlaceholder'; const PEER_B = '12D3KooWLexPlaceholder'; @@ -71,7 +75,7 @@ describe('ProtocolOutbox.enqueueFailure', () => { payload[0] = 9; entry.payload[1] = 8; - const pending = outbox.pendingFor(PEER_A); + const pending = outbox.list().filter((entry) => entry.peer === PEER_A); expect(Array.from(pending[0].payload)).toEqual([1, 2, 3]); pending[0].payload[2] = 7; @@ -128,7 +132,7 @@ describe('ProtocolOutbox.tryBeginAttempt / endAttempt', () => { }); }); -describe('ProtocolOutbox.due / pendingFor', () => { +describe('ProtocolOutbox.due / peer presence', () => { it('due returns entries whose nextAttemptAt is at or before now', () => { const { outbox } = fixture(); outbox.enqueueFailure(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1000); @@ -137,14 +141,86 @@ describe('ProtocolOutbox.due / pendingFor', () => { expect(outbox.due(expectedNext)).toHaveLength(1); }); - it('pendingFor returns all entries for a peer in firstFailureAt ascending order', () => { + it('bounds due snapshots in retry-time order', () => { + const { outbox } = fixture(); + outbox.enqueueFailure(PEER_A, PROTO, 'third', PAYLOAD, 'e', 3000); + outbox.enqueueFailure(PEER_A, PROTO, 'first', PAYLOAD, 'e', 1000); + outbox.enqueueFailure(PEER_A, PROTO, 'second', PAYLOAD, 'e', 2000); + const now = 3000 + DEFAULT_PROTOCOL_OUTBOX_BACKOFFS_MS[0]; + expect(outbox.duePage(now, 2).map((entry) => entry.messageId)).toEqual(['first', 'second']); + }); + + it('normalizes limits once and deterministically orders exact timestamp ties', () => { + const { outbox } = fixture(); + outbox.enqueueFailure(PEER_A, PROTO, 'z-last', PAYLOAD, 'e', 1000); + outbox.enqueueFailure(PEER_A, PROTO, 'a-first', PAYLOAD, 'e', 1000); + const now = 1000 + DEFAULT_PROTOCOL_OUTBOX_BACKOFFS_MS[0]; + + expect(outbox.duePage(now, 1.9).map((entry) => entry.messageId)).toEqual(['a-first']); + expect(outbox.duePage(now, Number.NaN).map((entry) => entry.messageId)).toEqual(['a-first', 'z-last']); + }); + + it('keeps legacy due-only stores compatible and sorts before applying the cap', () => { + const backing = new InMemoryProtocolOutboxStore(); + const entry = ( + messageId: string, + firstFailureAt: number, + ): ProtocolOutboxEntry => ({ + peer: PEER_A, + protocol: PROTO, + messageId, + payload: PAYLOAD, + attempts: 1, + firstFailureAt, + lastAttemptAt: firstFailureAt, + nextAttemptAt: 100, + lastError: 'offline', + }); + const newer = entry('a-newer-failure', 20); + const older = entry('z-older-failure', 10); + const legacyStore: ProtocolOutboxStore = { + enqueue: backing.enqueue.bind(backing), + markDelivered: backing.markDelivered.bind(backing), + hasEntry: backing.hasEntry.bind(backing), + hasPendingFor: backing.hasPendingFor.bind(backing), + due: () => [newer, older], + dropExpired: backing.dropExpired.bind(backing), + size: backing.size.bind(backing), + list: backing.list.bind(backing), + getEntry: backing.getEntry.bind(backing), + }; + const outbox = new ProtocolOutbox(legacyStore); + + expect(outbox.duePage(100, 1).map((candidate) => candidate.messageId)) + .toEqual(['z-older-failure']); + expect(outbox.due(100).map((candidate) => candidate.messageId)) + .toEqual(['z-older-failure', 'a-newer-failure']); + }); + + it('uses firstFailureAt before key ordering when nextAttemptAt ties', () => { + const store = new InMemoryProtocolOutboxStore({ backoffs: [50, 10] }); + const outbox = new ProtocolOutbox(store, { backoffs: [50, 10] }); + outbox.enqueueFailure(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'first', 0); + outbox.enqueueFailure(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'second', 90); + outbox.enqueueFailure(PEER_A, PROTO, 'a-newer-failure', PAYLOAD, 'first', 50); + + const due = outbox.duePage(100, 1); + expect(due).toHaveLength(1); + expect(due[0]).toMatchObject({ + messageId: 'z-older-failure', + firstFailureAt: 0, + nextAttemptAt: 100, + }); + }); + + it('hasPendingFor tracks peer rows without exposing a reconnect drain snapshot', () => { const { outbox } = fixture(); outbox.enqueueFailure(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2000); outbox.enqueueFailure(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1000); outbox.enqueueFailure(PEER_B, PROTO, MSG_1, PAYLOAD, 'e', 500); - const peerA = outbox.pendingFor(PEER_A); - expect(peerA.map((e) => e.messageId)).toEqual([MSG_1, MSG_2]); - expect(outbox.pendingFor(PEER_B)).toHaveLength(1); + expect(outbox.hasPendingFor(PEER_A)).toBe(true); + expect(outbox.hasPendingFor(PEER_B)).toBe(true); + expect(outbox.hasPendingFor('peer-c')).toBe(false); }); }); diff --git a/packages/core/test/publisher-extension.test.ts b/packages/core/test/publisher-extension.test.ts index a55f4d3ace..043c76b665 100644 --- a/packages/core/test/publisher-extension.test.ts +++ b/packages/core/test/publisher-extension.test.ts @@ -46,6 +46,11 @@ describe('DkgPublisherExtension', () => { '"42"^^', ); expect(escapeDkgRdfLiteral('a "quote"\nnext')).toBe('a \\"quote\\"\\nnext'); + // #416: non-ECHAR C0 controls (NUL, VT, unit-sep) + DEL must be escaped as + // \uXXXX, not passed through raw (raw controls = invalid N-Triples literal). + expect(escapeDkgRdfLiteral('n\u0000v\u000Bu\u001Fd\u007F')).toBe('n\\u0000v\\u000Bu\\u001Fd\\u007F'); + // ECHAR short forms are preserved (stable merkle output). + expect(escapeDkgRdfLiteral('t\tn\nr\rf\fb\b')).toBe('t\\tn\\nr\\rf\\fb\\b'); }); it('normalizes full quads without changing URI, literal, or blank-node RDF terms', () => { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index d231bbc57e..d99ceee5ab 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -5,5 +5,6 @@ "rootDir": "src", "composite": true }, - "include": ["src"] + "include": ["src"], + "references": [{ "path": "../rdf-utils" }] } diff --git a/packages/mcp-dkg/package.json b/packages/mcp-dkg/package.json index 5006833365..7b373ca3c1 100644 --- a/packages/mcp-dkg/package.json +++ b/packages/mcp-dkg/package.json @@ -42,6 +42,7 @@ "clean": "rm -rf dist" }, "dependencies": { + "@origintrail-official/dkg-rdf-utils": "workspace:*", "@modelcontextprotocol/sdk": "^1", "zod": "^3.25", "yaml": "^2.6.0" diff --git a/packages/mcp-dkg/src/tools/assertions.ts b/packages/mcp-dkg/src/tools/assertions.ts index 615f289d0d..a0894ec28c 100644 --- a/packages/mcp-dkg/src/tools/assertions.ts +++ b/packages/mcp-dkg/src/tools/assertions.ts @@ -15,6 +15,11 @@ * assertion name. */ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { + escapeRdfLiteral, + isRdfTerm as isSharedRdfTerm, + normalizeRdfObject as normalizeSharedRdfObject, +} from '@origintrail-official/dkg-rdf-utils'; import { z } from 'zod'; import type { DkgClient } from '../client.js'; import { DkgHttpError } from '../client.js'; @@ -105,12 +110,10 @@ function validateAssertionName(name: string): { valid: boolean; reason?: string /** * Object-term normalizer for the `dkg_knowledge_asset_create` one-shot `quads` - * path. Replicated verbatim from `@origintrail-official/dkg-core` - * `normalizeDkgPublisherObject` / `isDkgRdfTerm` / `escapeDkgRdfLiteral` - * (`packages/core/src/publisher-extension.ts:212-239`) — MCP does not depend on - * dkg-core (it is deliberately dependency-light; it inlines `validateAssertionName` - * and the share-warning constants for the same reason), and core does not export the - * normalizer from its package entry, so the rule is inlined here. OpenClaw + Hermes + * path. The complete dependency-free normalization boundary (term classification, + * literal escaping, and quote wrapping) lives in + * `@origintrail-official/dkg-rdf-utils`. MCP remains independent of the full + * dkg-core runtime. OpenClaw + Hermes * route the SAME create-tool `quads` shape through that core normalizer, so a bare * literal object must auto-quote identically across all three runtimes * (portable-agent parity). A value that is already an http(s)/urn/did URI, a blank @@ -120,33 +123,20 @@ function validateAssertionName(name: string): { valid: boolean; reason?: string * * DRIFT GUARD: the public contract of these three functions is pinned by a GOLDEN * conformance fixture in `packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts` - * whose expected strings are HAND-WRITTEN (not derived from this impl). If core's - * `normalizeDkgPublisherObject`/`escapeDkgRdfLiteral` changes, update this inline - * copy AND that fixture together (core carries a back-pointer to that test). + * whose expected strings are HAND-WRITTEN (not derived from this impl). * * Exported so the conformance test can pin them directly. */ export function isRdfTerm(value: string): boolean { - return ( - /^(?:https?:\/\/|urn:|did:)/i.test(value) || - value.startsWith('_:') || - value.startsWith('"') - ); + return isSharedRdfTerm(value); } export function escapeRdfLiteralBody(value: string): string { - return value - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/\t/g, '\\t') - .replace(/\f/g, '\\f') - .replace(/\x08/g, '\\b'); + return escapeRdfLiteral(value); } export function normalizeRdfObject(value: string): string { - return isRdfTerm(value) ? value : `"${escapeRdfLiteralBody(value)}"`; + return normalizeSharedRdfObject(value); } function resolveProject( diff --git a/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts b/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts index 538613346c..5705f71e1e 100644 --- a/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts +++ b/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts @@ -65,6 +65,11 @@ const GOLDEN: Array<{ input: string; expected: string; note: string }> = [ { input: 'form\ffeed', expected: '"form\\ffeed"', note: 'form-feed escaped' }, { input: 'bs\bhere', expected: '"bs\\bhere"', note: 'backspace escaped' }, { input: 'a"b\nc', expected: '"a\\"b\\nc"', note: 'quote + newline together' }, + // ── non-ECHAR control chars (#416): must become \uXXXX, not pass through raw ── + { input: 'nul\u0000x', expected: '"nul\\u0000x"', note: 'NUL → \\u0000 UCHAR' }, + { input: 'vt\u000Bx', expected: '"vt\\u000Bx"', note: 'vertical tab (0x0B) → \\u000B' }, + { input: 'us\u001Fx', expected: '"us\\u001Fx"', note: 'unit separator (0x1F) → \\u001F' }, + { input: 'del\u007Fx', expected: '"del\\u007Fx"', note: 'DEL (0x7F) → \\u007F' }, ]; describe('rdf-object-normalization conformance (dkg_knowledge_asset_create quads — cross-adapter parity)', () => { @@ -90,5 +95,7 @@ describe('rdf-object-normalization conformance (dkg_knowledge_asset_create quads it('escapeRdfLiteralBody escapes the N-Triples ECHAR set (body only, no surrounding quotes)', () => { expect(escapeRdfLiteralBody('a\\b"c\nd\re\tf\fg\bh')).toBe('a\\\\b\\"c\\nd\\re\\tf\\fg\\bh'); expect(escapeRdfLiteralBody('plain')).toBe('plain'); + // #416: non-ECHAR C0 controls + DEL become \uXXXX (uppercase, 4-digit). + expect(escapeRdfLiteralBody('a\u0000b\u000Bc\u007Fd')).toBe('a\\u0000b\\u000Bc\\u007Fd'); }); }); diff --git a/packages/mcp-dkg/tsconfig.json b/packages/mcp-dkg/tsconfig.json index d231bbc57e..d99ceee5ab 100644 --- a/packages/mcp-dkg/tsconfig.json +++ b/packages/mcp-dkg/tsconfig.json @@ -5,5 +5,6 @@ "rootDir": "src", "composite": true }, - "include": ["src"] + "include": ["src"], + "references": [{ "path": "../rdf-utils" }] } diff --git a/packages/node-ui/src/db.ts b/packages/node-ui/src/db.ts index 645cae4059..eba2e4edc0 100644 --- a/packages/node-ui/src/db.ts +++ b/packages/node-ui/src/db.ts @@ -2774,12 +2774,19 @@ export class SqliteProtocolOutboxStore implements ProtocolOutboxStore { return row !== undefined; } - pendingFor(peer: string): ProtocolOutboxEntry[] { + hasPendingFor(peer: string): boolean { + return this.db.prepare('SELECT 1 FROM protocol_outbox WHERE peer_id = ? LIMIT 1').get(peer) !== undefined; + } + + due(now: number): ProtocolOutboxEntry[] { const rows = this.db .prepare( - `SELECT * FROM protocol_outbox WHERE peer_id = ? ORDER BY first_failure_at ASC`, + `SELECT * FROM protocol_outbox + WHERE next_attempt_at <= ? + ORDER BY next_attempt_at ASC, first_failure_at ASC, + peer_id ASC, protocol ASC, message_id ASC`, ) - .all(peer) as Array<{ + .all(now) as Array<{ peer_id: string; protocol: string; message_id: string; @@ -2793,12 +2800,16 @@ export class SqliteProtocolOutboxStore implements ProtocolOutboxStore { return rows.map(SqliteProtocolOutboxStore.rowToEntry); } - due(now: number): ProtocolOutboxEntry[] { + duePage(now: number, limit: number): ProtocolOutboxEntry[] { const rows = this.db .prepare( - `SELECT * FROM protocol_outbox WHERE next_attempt_at <= ?`, + `SELECT * FROM protocol_outbox + WHERE next_attempt_at <= ? + ORDER BY next_attempt_at ASC, first_failure_at ASC, + peer_id ASC, protocol ASC, message_id ASC + LIMIT ?`, ) - .all(now) as Array<{ + .all(now, limit) as Array<{ peer_id: string; protocol: string; message_id: string; diff --git a/packages/node-ui/test/messenger-stores.test.ts b/packages/node-ui/test/messenger-stores.test.ts index 21efc6e152..9b8f59feac 100644 --- a/packages/node-ui/test/messenger-stores.test.ts +++ b/packages/node-ui/test/messenger-stores.test.ts @@ -180,7 +180,7 @@ describe('SqliteProtocolOutboxStore', () => { payload[0] = 9; entry.payload[1] = 8; - const pending = store.pendingFor(PEER_A); + const pending = store.list().filter((entry) => entry.peer === PEER_A); expect(Array.from(pending[0].payload)).toEqual([1, 2, 3]); pending[0].payload[2] = 7; @@ -207,13 +207,14 @@ describe('SqliteProtocolOutboxStore', () => { expect(store.markDelivered(PEER_A, PROTO, MSG_1)).toBe(false); }); - it('pendingFor returns entries sorted by firstFailureAt ascending', () => { + it('hasPendingFor reports whether a peer owns any durable row', () => { const store = new SqliteProtocolOutboxStore(db); store.enqueue(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2000); store.enqueue(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1000); store.enqueue(PEER_B, PROTO, MSG_1, PAYLOAD, 'e', 500); - expect(store.pendingFor(PEER_A).map((e) => e.messageId)).toEqual([MSG_1, MSG_2]); - expect(store.pendingFor(PEER_B)).toHaveLength(1); + expect(store.hasPendingFor(PEER_A)).toBe(true); + expect(store.hasPendingFor(PEER_B)).toBe(true); + expect(store.hasPendingFor('peer-c')).toBe(false); }); it('due returns entries with nextAttemptAt <= now', () => { @@ -223,6 +224,44 @@ describe('SqliteProtocolOutboxStore', () => { expect(store.due(1_005_000)).toHaveLength(1); }); + it('due applies a stable database-level batch limit', () => { + const store = new SqliteProtocolOutboxStore(db, { backoffFor: () => 5_000 }); + store.enqueue(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2_000); + store.enqueue(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1_000); + expect(store.duePage(10_000, 1).map((entry) => entry.messageId)).toEqual([MSG_1]); + expect(store.duePage(10_000, 0)).toEqual([]); + }); + + it('due deterministically breaks exact timestamp ties by peer/protocol/message id', () => { + const store = new SqliteProtocolOutboxStore(db, { backoffFor: () => 5_000 }); + store.enqueue(PEER_B, PROTO, MSG_2, PAYLOAD, 'e', 1_000); + store.enqueue(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 1_000); + store.enqueue(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1_000); + + expect(store.duePage(6_000, 3).map((entry) => [entry.peer, entry.messageId])).toEqual([ + [PEER_B, MSG_2], + [PEER_A, MSG_1], + [PEER_A, MSG_2], + ]); + }); + + it('uses firstFailureAt before key ordering when nextAttemptAt ties', () => { + const store = new SqliteProtocolOutboxStore(db, { + backoffFor: (attempts) => attempts === 1 ? 50 : 10, + }); + store.enqueue(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'first', 0); + store.enqueue(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'second', 90); + store.enqueue(PEER_A, PROTO, 'a-newer-failure', PAYLOAD, 'first', 50); + + const due = store.duePage(100, 1); + expect(due).toHaveLength(1); + expect(due[0]).toMatchObject({ + messageId: 'z-older-failure', + firstFailureAt: 0, + nextAttemptAt: 100, + }); + }); + it('dropExpired removes entries older than maxAgeMs and returns them', () => { const store = new SqliteProtocolOutboxStore(db, { maxAgeMs: 60_000, @@ -259,7 +298,7 @@ describe('SqliteProtocolOutboxStore', () => { db.close(); db = new DashboardDB({ dataDir: dir }); const reopened = new SqliteProtocolOutboxStore(db); - const pending = reopened.pendingFor(PEER_A); + const pending = reopened.list().filter((entry) => entry.peer === PEER_A); expect(pending).toHaveLength(1); expect(pending[0].lastError).toBe('crash-before-delivery'); expect(Array.from(pending[0].payload)).toEqual(Array.from(PAYLOAD)); diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index d379456193..fc889ac964 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -1,9 +1,9 @@ -import type { Quad, TripleStore } from '@origintrail-official/dkg-storage'; +import type { Quad, SharedMemoryGraphScope, TripleStore } from '@origintrail-official/dkg-storage'; import type { ChainAdapter, OnChainPublishResult, AddBatchToContextGraphParams } from '@origintrail-official/dkg-chain'; import { enrichEvmError } from '@origintrail-official/dkg-chain'; import type { EventBus, OperationContext } from '@origintrail-official/dkg-core'; import { DKGEvent, Logger, createOperationContext, sha256, encodeWorkspacePublishRequest, encodeEncryptedWorkspacePayload, encryptWorkspacePayload, contextGraphDataUri, contextGraphDataGraphUri, contextGraphMetaUri, contextGraphAssertionUri, contextGraphLayerUri, MemoryLayer, assertionLifecycleUri, contextGraphSubGraphUri, contextGraphSubGraphMetaUri, SYSTEM_CONTEXT_GRAPHS, validateSubGraphName, isSafeIri, assertSafeIri, assertSafeRdfTerm, assertQuadLiteralsMutf8Safe, DKG_GOSSIP_MAX_MESSAGE_BYTES, SwmGossipPayloadTooLargeError, STORAGE_ACK_MAX_STAGING_BYTES, type Ed25519Keypair, buildAuthorAttestationTypedData, buildUpdateAuthorAttestationTypedData, AUTHOR_SCHEME_VERSION_V1, TrustLevel, TRUST_LEVEL_PREDICATE, assertNoUserAuthoredTrustLevelQuads, buildTrustLevelQuads, isTrustLevelQuad, isSwmMerkleExcludedQuad, WORKSPACE_OWNER_PREDICATE, DKG_ENTITY, DKG_ROOT_ENTITY_LEGACY, ENTITY_PRED_ALT, parseAssertionSealQuads, ASSERTION_SEAL_PREDICATES, sharedMemoryReadBothFilter, DKG_ONTOLOGY } from '@origintrail-official/dkg-core'; -import { GraphManager, PrivateContentStore, loadSelectedSharedMemoryQuads } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, loadSharedMemoryQuadsForScope, loadSelectedSharedMemoryQuads, resolveSharedMemoryScopeGraphs } from '@origintrail-official/dkg-storage'; import { DEFAULT_PUBLISH_EPOCHS, MAX_PUBLISH_EPOCHS, type Publisher, type PublishOptions, type PublishResult, type KAManifestEntry, type PhaseCallback, type V10CoreNodeACK, type V10ACKProviderParams, type V10ACKProviderObject, type LegacyV10ACKProvider } from './publisher.js'; import { skolemizeByEntity } from './auto-partition.js'; import { withKeyedLocks } from './keyed-lock.js'; @@ -1556,9 +1556,19 @@ export class DKGPublisher implements Publisher { * the existing allocate-at-publish behavior. */ reservedKaId?: bigint; + /** Explicit graph-family boundary; named lifecycles exclude bucket and siblings. */ + sharedMemoryScope?: SharedMemoryGraphScope; }, ): Promise { const ctx = options?.operationCtx ?? createOperationContext('publishFromSWM'); + const sharedMemoryScope: SharedMemoryGraphScope = options?.sharedMemoryScope + ?? { kind: 'complete-family' }; + if (sharedMemoryScope.kind === 'named-lifecycle' && options?.clearSharedMemoryAfter === true) { + throw new Error( + 'clearSharedMemoryAfter cannot be combined with a named-lifecycle shared-memory scope; ' + + 'use complete-family scope for an explicit family-wide clear', + ); + } // Guard: VM publishing requires an on-chain registered context graph. // Skip for mock/none chains (unit tests) — only enforce on real chains. @@ -1601,14 +1611,21 @@ export class DKGPublisher implements Publisher { const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, options?.subGraphName); - const quads = await loadSelectedSharedMemoryQuads(this.store, swmGraph, selection, { - quadFilter: (q) => !isSwmMerkleExcludedQuad(q), - rootEntitiesErrorMessage: ({ inputCount, hadInput }) => ( + const loadOptions = { + quadFilter: (q: Quad) => !isSwmMerkleExcludedQuad(q), + rootEntitiesErrorMessage: ({ inputCount, hadInput }: { inputCount: number; hadInput: boolean }) => ( hadInput ? `No valid rootEntities provided (all ${inputCount} entries failed IRI validation)` : `No rootEntities provided for context graph ${contextGraphId}` ), - }); + }; + const quads = await loadSharedMemoryQuadsForScope( + this.store, + swmGraph, + selection, + sharedMemoryScope, + loadOptions, + ); if (quads.length === 0) { throw new Error(`No quads in shared memory for context graph ${contextGraphId} matching selection`); @@ -1712,58 +1729,17 @@ export class DKGPublisher implements Publisher { // `ctxGraphId ?? chainCgId`. const targetCgId = ctxGraphId ?? chainCgId; if (targetCgId && publishResult.status === 'confirmed' && publishResult.onChainResult) { - // V10 publishDirect already registers the KC to the context graph - // via an internal call to ContextGraphs.registerKnowledgeAsset - // (Hub-authorized only — EOAs cannot call it directly). The legacy - // V9 flow required a separate addBatchToContextGraph tx; that path - // is no longer available. Attempt the explicit verify call as a - // fallback for non-V10 chains, but treat "Only Contracts in Hub" - // rejections as success (V10 already handled it). - let registered = false; - if (typeof this.chain.verify === 'function') { - let participantSigs = options?.contextGraphSignatures ?? []; - if (participantSigs.length === 0 && typeof this.chain.signMessage === 'function') { - const identityId = this.publisherNodeIdentityId; - if (identityId > 0n) { - const digest = ethers.solidityPackedKeccak256( - ['uint256', 'bytes32'], - [BigInt(targetCgId), ethers.hexlify(publishResult.merkleRoot)], - ); - const sig = await this.chain.signMessage(ethers.getBytes(digest)); - participantSigs = [{ identityId, ...sig }]; - } - } + // V10 publishDirect already registered the KC to the context graph + // inside publishDirect, via a Hub-authorized internal call to + // ContextGraphs.registerKnowledgeAsset (EOAs cannot call it directly), + // which emits KnowledgeAssetRegisteredToContextGraph. The legacy V9 + // explicit chain.verify() fallback that used to run here always reverted + // on V10 ("Only Contracts in Hub" / CALL_EXCEPTION) — i.e. a doomed + // on-chain call plus an estimateGas round-trip, and serializer occupancy, + // on EVERY confirmed publish (#1575). Registration is already done, so + // skip the verify attempt entirely and proceed to the data promotion. + this.log.debug(ctx, `V10 auto-registered KC to context graph ${targetCgId}; explicit verify skipped`); - const sortedSigs = [...participantSigs] - .sort((a, b) => (a.identityId < b.identityId ? -1 : a.identityId > b.identityId ? 1 : 0)) - .filter((s, i, arr) => i === 0 || s.identityId !== arr[i - 1].identityId); - - try { - const txResult = await this.chain.verify({ - contextGraphId: BigInt(targetCgId), - batchId: publishResult.onChainResult.batchId, - merkleRoot: publishResult.merkleRoot, - signerSignatures: sortedSigs, - }); - if (txResult && typeof txResult === 'object' && 'success' in txResult && txResult.success) { - registered = true; - this.log.info(ctx, `Batch ${publishResult.onChainResult.batchId} verified on context graph ${targetCgId}`); - } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - // V10 publishDirect handles registration internally via a - // Hub-authorized call. Any revert here (typically - // "Only Contracts in Hub" / CALL_EXCEPTION) means the - // explicit verify path is not applicable — treat as success. - registered = true; - this.log.info(ctx, `Explicit verify not needed (V10 auto-registered): ${msg.slice(0, 120)}`); - } - } else { - registered = true; - this.log.info(ctx, `No verify function on chain adapter — assuming V10 auto-registration for context graph ${targetCgId}`); - } - - if (registered) { const ctxDataGraph = contextGraphDataUri(contextGraphId, targetCgId); const ctxMetaGraph = contextGraphMetaUri(contextGraphId, targetCgId); const defaultDataGraph = this.graphManager.dataGraphUri(contextGraphId); @@ -1883,7 +1859,6 @@ export class DKGPublisher implements Publisher { this.log.info(ctx, `Promoted ${publishResult.kaManifest.length} KAs from default graph to context graph ${targetCgId}`); } }); - } } // SWM cleanup: ALWAYS remove published triples from SWM after chain confirmation. @@ -1891,7 +1866,13 @@ export class DKGPublisher implements Publisher { // clearSharedMemoryAfter controls only whether the REMAINING unpublished triples are also cleared. if (publishResult.status === 'confirmed') { const kaMap = skolemizeByEntity(quads); - await this.clearPublishedSwmRoots(contextGraphId, [...kaMap.keys()], options?.subGraphName, ctx); + await this.clearPublishedSwmRoots( + contextGraphId, + [...kaMap.keys()], + options?.subGraphName, + ctx, + sharedMemoryScope, + ); // If clearSharedMemoryAfter is explicitly true, also clear any remaining unpublished content. // Default is false: unpublished entities stay in SWM for future publishes. if (options?.clearSharedMemoryAfter === true) { @@ -5490,12 +5471,31 @@ export class DKGPublisher implements Publisher { rootEntities: string[], subGraphName: string | undefined, ctx: OperationContext, + scope: SharedMemoryGraphScope = { kind: 'complete-family' }, ): Promise { if (rootEntities.length === 0) return; + const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, subGraphName); + await this.clearPublishedSwmRootsInGraphs( + contextGraphId, + rootEntities, + subGraphName, + ctx, + await resolveSharedMemoryScopeGraphs(this.store, swmGraph, scope), + scope.kind === 'complete-family' ? 'always' : 'when-no-share-remains', + ); + } + + private async clearPublishedSwmRootsInGraphs( + contextGraphId: string, + rootEntities: string[], + subGraphName: string | undefined, + ctx: OperationContext, + swmGraphsForClear: string[], + metadataPolicy: 'always' | 'when-no-share-remains', + ): Promise { const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, subGraphName); const swmMetaGraph = this.graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); const swmOwnershipKey = subGraphName ? `${contextGraphId}\0${subGraphName}` : contextGraphId; - const swmGraphsForClear = await this.swmGraphsUnder(swmGraph); let ownerDeletedTotal = 0; for (const rootEntity of rootEntities) { for (const g of swmGraphsForClear) { @@ -5505,12 +5505,40 @@ export class DKGPublisher implements Publisher { graph: g, subject: rootEntity, predicate: WORKSPACE_OWNER_PREDICATE, }); } - const ownerDeleted = await this.store.deleteByPattern({ - graph: swmMetaGraph, subject: rootEntity, predicate: WORKSPACE_OWNER_PREDICATE, - }); - ownerDeletedTotal += ownerDeleted; - await this.deleteMetaForRoot(swmMetaGraph, rootEntity); - this.sharedMemoryOwnedEntities.get(swmOwnershipKey)?.delete(rootEntity); + } + // A root-keyed owner row can only remain live while some SWM family graph + // still contains that root. Reconcile the complete selected root set with + // one family-wide read rather than one independent scan per root. + const rootsWithRemainingShares = new Set(); + if (metadataPolicy === 'when-no-share-remains') { + const remaining = await loadSelectedSharedMemoryQuads( + this.store, + swmGraph, + { rootEntities }, + { querySource: 'publisher.clearPublishedNamedKnowledgeAssetRoots.reconcileOwnership' }, + ); + for (const quad of remaining) { + for (const rootEntity of rootEntities) { + if ( + quad.subject === rootEntity + || quad.subject.startsWith(`${rootEntity}/.well-known/genid/`) + ) { + rootsWithRemainingShares.add(rootEntity); + } + } + } + } + for (const rootEntity of rootEntities) { + const shouldClearRootMetadata = metadataPolicy === 'always' + || !rootsWithRemainingShares.has(rootEntity); + if (shouldClearRootMetadata) { + const ownerDeleted = await this.store.deleteByPattern({ + graph: swmMetaGraph, subject: rootEntity, predicate: WORKSPACE_OWNER_PREDICATE, + }); + ownerDeletedTotal += ownerDeleted; + await this.deleteMetaForRoot(swmMetaGraph, rootEntity); + this.sharedMemoryOwnedEntities.get(swmOwnershipKey)?.delete(rootEntity); + } } if (ownerDeletedTotal > 0) { this.log.info(ctx, `Cleared ${ownerDeletedTotal} published SWM triple(s) after confirmed publish`); diff --git a/packages/publisher/test/dkg-publisher.test.ts b/packages/publisher/test/dkg-publisher.test.ts index 2a7fc9cc7f..667f07e94a 100644 --- a/packages/publisher/test/dkg-publisher.test.ts +++ b/packages/publisher/test/dkg-publisher.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, beforeAll, afterAll, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, beforeAll, afterAll, afterEach, vi } from 'vitest'; import { OxigraphStore } from '@origintrail-official/dkg-storage'; import { EVMChainAdapter } from '@origintrail-official/dkg-chain'; import { @@ -336,6 +336,30 @@ describe('DKGPublisher', () => { expect(result.onChainResult!.endKAId).toBeDefined(); }); + it('does NOT call chain.verify after a confirmed V10 SWM publish (#1575)', async () => { + const quad = q(ENTITY, 'http://schema.org/name', '"ImageBot"'); + await publisher.share(CONTEXT_GRAPH, [quad], { + publisherPeerId: 'peer-no-legacy-verify', + localOnly: true, + }); + await seedContextGraphRegistration(store, CONTEXT_GRAPH); + + const verifySpy = vi.spyOn(chain, 'verify'); + const result = await publisher.publishFromSharedMemory(CONTEXT_GRAPH, 'all', { + onChainContextGraphId: CONTEXT_GRAPH, + precomputedAttestation: await buildSeal({ + quads: [{ ...quad, graph: GRAPH }], + author: _author, + contextGraphId: CONTEXT_GRAPH, + ctx: { provider: _provider, kav10Address: _kav10Address }, + }), + }); + + expect(result.status).toBe('confirmed'); + expect(result.onChainResult).toBeDefined(); + expect(verifySpy).not.toHaveBeenCalled(); + }); + it('generates address-based UAL format', async () => { const result = await publishWS({ contextGraphId: CONTEXT_GRAPH, diff --git a/packages/publisher/test/shared-memory-publish-boundary.test.ts b/packages/publisher/test/shared-memory-publish-boundary.test.ts index 880e09715f..5990f6d251 100644 --- a/packages/publisher/test/shared-memory-publish-boundary.test.ts +++ b/packages/publisher/test/shared-memory-publish-boundary.test.ts @@ -4,6 +4,7 @@ import { TRUST_LEVEL_PREDICATE, TrustLevel, TypedEventBus, + createOperationContext, encodeWorkspacePublishRequest, generateEd25519Keypair, DKG_ENTITY, @@ -23,6 +24,8 @@ const CONTEXT_GRAPH_URI = `did:dkg:context-graph:${CONTEXT_GRAPH}`; const SWM_GRAPH = `did:dkg:context-graph:${CONTEXT_GRAPH}/_shared_memory`; const SWM_META_GRAPH = `did:dkg:context-graph:${CONTEXT_GRAPH}/_shared_memory_meta`; const PER_KA_SWM_GRAPH = `${SWM_GRAPH}/0x1111111111111111111111111111111111111111/1`; +const SAME_AUTHOR_SIBLING_SWM_GRAPH = `${SWM_GRAPH}/0x1111111111111111111111111111111111111111/2`; +const FOREIGN_PER_KA_SWM_GRAPH = `${SWM_GRAPH}/0x2222222222222222222222222222222222222222/9`; const ONTOLOGY_GRAPH = 'did:dkg:context-graph:ontology'; const ON_CHAIN_ID_PREDICATE = 'https://dkg.network/ontology#ContextGraphOnChainId'; const WORKSPACE_OWNER_PREDICATE = 'http://dkg.io/ontology/workspaceOwner'; @@ -66,7 +69,7 @@ async function makeRealPublisher(chain = new NoChainAdapter()) { return { publisher, store }; } -async function makePublisher(chain = new NoChainAdapter()) { +async function makePublisher(chain = new NoChainAdapter(), status: PublishResult['status'] = 'tentative') { const { publisher, store } = await makeRealPublisher(chain); const publishResult: PublishResult = { kaId: 1n, @@ -79,7 +82,7 @@ async function makePublisher(chain = new NoChainAdapter()) { privateTripleCount: 0, }, ], - status: 'tentative', + status, publicQuads: [], }; const publishSpy = recorder(async (..._args: Parameters) => publishResult); @@ -206,6 +209,152 @@ describe('publishFromSharedMemory multi-root selection (OT-RFC-44 / Design B: on ]); }); + it('named-KA scope excludes a foreign share with the same subject IRI', async () => { + const { publisher, store, publishSpy } = await makePublisher(); + await store.insert([ + q('urn:test:root:one', 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q('urn:test:root:one', 'http://schema.org/name', '"same-author-sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + q('urn:test:root:one', 'http://schema.org/name', '"foreign"', FOREIGN_PER_KA_SWM_GRAPH), + q('urn:test:root:one', 'http://schema.org/name', '"legacy-bucket"', SWM_GRAPH), + ]); + + await publisher.publishFromSharedMemory( + CONTEXT_GRAPH, + { rootEntities: ['urn:test:root:one'] }, + { + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }, + ); + + expect(publishSpy.calls[0][0].quads).toEqual([ + { subject: 'urn:test:root:one', predicate: 'http://schema.org/name', object: '"local"', graph: '' }, + ]); + }); + + it('confirmed exact cleanup removes stale ownership when the local KA was the last share', async () => { + const { publisher, store } = await makePublisher(new NoChainAdapter(), 'confirmed'); + const root = 'urn:test:root:one'; + await store.insert([ + q(root, 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q(root, WORKSPACE_OWNER_PREDICATE, '"peer-a"', SWM_META_GRAPH), + ]); + + await publisher.publishFromSharedMemory(CONTEXT_GRAPH, { rootEntities: [root] }, { + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }); + + expect(await store.deleteByPattern({ graph: PER_KA_SWM_GRAPH, subject: root })).toBe(0); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: root })).toBe(0); + const owners = await (publisher as any).sharedMemoryOwnersForPromotion( + CONTEXT_GRAPH, undefined, CONTEXT_GRAPH, [root], + ); + expect(owners.size).toBe(0); + }); + + it('confirmed exact cleanup drains only local data and preserves foreign share ownership', async () => { + const { publisher, store } = await makePublisher(new NoChainAdapter(), 'confirmed'); + const root = 'urn:test:root:one'; + await store.insert([ + q(root, 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q(root, 'http://schema.org/name', '"same-author-sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + q(root, 'http://schema.org/name', '"foreign"', FOREIGN_PER_KA_SWM_GRAPH), + q(root, WORKSPACE_OWNER_PREDICATE, '"peer-foreign"', SWM_META_GRAPH), + ]); + + await publisher.publishFromSharedMemory(CONTEXT_GRAPH, { rootEntities: [root] }, { + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }); + + expect(await store.deleteByPattern({ graph: PER_KA_SWM_GRAPH, subject: root })).toBe(0); + expect(await store.deleteByPattern({ graph: SAME_AUTHOR_SIBLING_SWM_GRAPH, subject: root })).toBe(1); + expect(await store.deleteByPattern({ graph: FOREIGN_PER_KA_SWM_GRAPH, subject: root })).toBe(1); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: root })).toBe(1); + }); + + it('batches multi-root exact-cleanup metadata reconciliation into one family read', async () => { + const { publisher, store } = await makePublisher(new NoChainAdapter(), 'confirmed'); + const consumedOnly = 'urn:test:root:consumed-only'; + const stillShared = 'urn:test:root:still-shared'; + await store.insert([ + q(consumedOnly, 'http://schema.org/name', '"local-one"', PER_KA_SWM_GRAPH), + q(stillShared, 'http://schema.org/name', '"local-two"', PER_KA_SWM_GRAPH), + q(`${stillShared}/.well-known/genid/1`, 'http://schema.org/name', '"sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + q(consumedOnly, WORKSPACE_OWNER_PREDICATE, '"peer-a"', SWM_META_GRAPH), + q(stillShared, WORKSPACE_OWNER_PREDICATE, '"peer-a"', SWM_META_GRAPH), + ]); + const originalQuery = store.query.bind(store); + let familyReads = 0; + store.query = async (...args) => { + if (args[1]?.source === 'publisher.clearPublishedNamedKnowledgeAssetRoots.reconcileOwnership') { + familyReads += 1; + } + return originalQuery(...args); + }; + + await publisher.clearPublishedSwmRoots( + CONTEXT_GRAPH, + [consumedOnly, stillShared], + undefined, + createOperationContext('test'), + { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + ); + + expect(familyReads).toBe(1); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: consumedOnly })).toBe(0); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: stillShared })).toBe(1); + }); + + it('rejects a family-wide remaining clear for an exact named lifecycle', async () => { + const { publisher, store, publishSpy } = await makePublisher(new NoChainAdapter(), 'confirmed'); + await store.insert([ + q('urn:test:root:one', 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q('urn:test:root:two', 'http://schema.org/name', '"sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + ]); + + await expect(publisher.publishFromSharedMemory( + CONTEXT_GRAPH, + { rootEntities: ['urn:test:root:one'] }, + { + clearSharedMemoryAfter: true, + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }, + )).rejects.toThrow(/cannot be combined with a named-lifecycle/); + + expect(publishSpy.calls).toHaveLength(0); + expect(await store.deleteByPattern({ graph: PER_KA_SWM_GRAPH })).toBe(1); + expect(await store.deleteByPattern({ graph: SAME_AUTHOR_SIBLING_SWM_GRAPH })).toBe(1); + }); + it('loads selected data root plus generated private-CG catalog root and threads trusted floor', async () => { const { publisher, store, publishSpy } = await makePublisher(privatePolicyChain()); const cgDid = `did:dkg:context-graph:${CONTEXT_GRAPH}`; diff --git a/packages/query/src/dkg-query-engine.ts b/packages/query/src/dkg-query-engine.ts index aafc504419..63306a2dc2 100644 --- a/packages/query/src/dkg-query-engine.ts +++ b/packages/query/src/dkg-query-engine.ts @@ -377,7 +377,8 @@ export class DKGQueryEngine implements QueryEngine { // Per-KA VM: read-both the published per-KA …/_verifiable_memory/{addr}/{number} + root. const vmGraphsInc = await this.discoverGraphsByPrefix(`${dataGraph}/_verifiable_memory/`); const dataSparql = vmGraphsInc.length > 0 - ? (wrapWithGraphUnion(sparql, [dataGraph, ...vmGraphsInc]) ?? wrapWithGraph(sparql, dataGraph)) + ? (this.wrapVerifiableMemoryGraphSet(sparql, [dataGraph, ...vmGraphsInc]) + ?? wrapWithGraph(sparql, dataGraph)) : wrapWithGraph(sparql, dataGraph); // Per-KA SWM: union the discovered …/_shared_memory/{addr}/{number} graphs. const swmGraphs = await this.discoverGraphsByPrefix(`${sharedMemoryGraph}/`); @@ -399,7 +400,8 @@ export class DKGQueryEngine implements QueryEngine { // Per-KA VM: read-both the published per-KA …/_verifiable_memory/{addr}/{number} + root. const vmGraphs = await this.discoverGraphsByPrefix(`${dataGraph}/_verifiable_memory/`); effectiveSparql = vmGraphs.length > 0 - ? (wrapWithGraphUnion(sparql, [dataGraph, ...vmGraphs]) ?? wrapWithGraph(sparql, dataGraph)) + ? (this.wrapVerifiableMemoryGraphSet(sparql, [dataGraph, ...vmGraphs]) + ?? wrapWithGraph(sparql, dataGraph)) : wrapWithGraph(sparql, dataGraph); } } @@ -587,9 +589,23 @@ export class DKGQueryEngine implements QueryEngine { return this.execAndNormalize(wrapWithGraph(effectiveSparql, allGraphs[0])); } + if (view === 'verifiable-memory') { + const rewritten = this.wrapVerifiableMemoryGraphSet(effectiveSparql, allGraphs); + if (rewritten !== null) return this.execAndNormalize(rewritten); + } + return this.queryMultipleGraphs(effectiveSparql, allGraphs); } + /** Canonical graph rewrite for root + per-KA/per-cgId verifiable-memory reads. */ + private wrapVerifiableMemoryGraphSet(sparql: string, graphs: string[]): string | null { + if (graphs.length === 0) return sparql; + if (graphs.length === 1) return wrapWithGraph(sparql, graphs[0]); + return wrapWithDeduplicatedGraphValues(sparql, graphs) + ?? wrapWithGraphValues(sparql, graphs) + ?? wrapWithGraphUnion(sparql, graphs); + } + private async queryMultipleGraphs(sparql: string, graphs: string[]): Promise { if (graphs.length === 0) return { bindings: [] }; if (graphs.length === 1) { @@ -2450,6 +2466,86 @@ function wrapWithGraphUnion(sparql: string, graphUris: string[]): string | null * view-specific name so a user query is astronomically unlikely to bind it; * a collision is nonetheless detected and declined (never silently clamped). */ const VIEW_GRAPH_SENTINEL = '?__dkgViewGraph'; +const DEDUP_GRAPH_SENTINEL = '?__dkgDedupGraph'; +const DEDUP_RANK_SENTINEL = '?__dkgDedupRank'; +const DEDUP_PRIOR_GRAPH_SENTINEL = '?__dkgDedupPriorGraph'; +const DEDUP_PRIOR_RANK_SENTINEL = '?__dkgDedupPriorRank'; + +function wrapWithProjectedGraphSubselect( + sparql: string, + graphUris: string[], + helperVariables: string[], + buildGraphPattern: (inner: string, graphs: string[]) => string, + acceptsInner: (inner: string) => boolean = () => true, +): string | null { + if (hasGraphClause(sparql)) return sparql; + if (graphUris.length === 0) return sparql; + + const braceStart = findWhereBraceStart(sparql); + if (braceStart === -1) return null; + const braceEnd = findMatchingCloseBrace(sparql, braceStart); + if (braceEnd === -1) return null; + + const before = sparql.slice(0, braceStart + 1); + const inner = sparql.slice(braceStart + 1, braceEnd); + const after = sparql.slice(braceEnd); + const graphs = [...new Set(graphUris)]; + + if (graphs.length === 1) { + return `${before} GRAPH <${assertSafeIri(graphs[0])}> { ${inner} } ${after}`; + } + if (/\bUNION\b/i.test(inner) || !acceptsInner(inner)) return null; + + const helperNames = new Set(helperVariables.map((variable) => variable.slice(1))); + if (collectQueryVariables(sparql).some((variable) => helperNames.has(variable.slice(1)))) { + return null; + } + + const innerVars = collectQueryVariables(inner); + if (innerVars.length === 0) return null; + + const graphPattern = buildGraphPattern(inner, graphs); + return `${before} { SELECT ${innerVars.join(' ')} WHERE { ${graphPattern} } } ${after}`; +} + +/** + * Run one graph pattern across an ordered graph set while suppressing only a + * solution mapping already produced by an earlier graph. The comparison uses + * every variable bound by the caller's inner pattern, before the caller's + * projection runs. Thus an identical mirrored triple is emitted once, while + * distinct triples that both project to the same `?s` still produce two rows. + * + * Helper graph/rank variables are hidden inside a sub-SELECT, preserving + * `SELECT *` and caller DISTINCT semantics. Unsupported/colliding query shapes + * return null so the existing generic multi-graph fallback remains available. + */ +function wrapWithDeduplicatedGraphValues(sparql: string, graphUris: string[]): string | null { + return wrapWithProjectedGraphSubselect( + sparql, + graphUris, + [ + DEDUP_GRAPH_SENTINEL, + DEDUP_RANK_SENTINEL, + DEDUP_PRIOR_GRAPH_SENTINEL, + DEDUP_PRIOR_RANK_SENTINEL, + ], + (inner, graphs) => { + const rows = graphs + .map((graph, rank) => `(<${assertSafeIri(graph)}> ${rank})`) + .join(' '); + return [ + `VALUES (${DEDUP_GRAPH_SENTINEL} ${DEDUP_RANK_SENTINEL}) { ${rows} }`, + `GRAPH ${DEDUP_GRAPH_SENTINEL} { ${inner} }`, + 'FILTER NOT EXISTS {', + ` VALUES (${DEDUP_PRIOR_GRAPH_SENTINEL} ${DEDUP_PRIOR_RANK_SENTINEL}) { ${rows} }`, + ` FILTER (${DEDUP_PRIOR_RANK_SENTINEL} < ${DEDUP_RANK_SENTINEL})`, + ` GRAPH ${DEDUP_PRIOR_GRAPH_SENTINEL} { ${inner} }`, + '}', + ].join(' '); + }, + isDedupSafeBasicGraphPattern, + ); +} /** * Wrap a query so it runs over a set of named graphs in ONE execution using a @@ -2480,52 +2576,53 @@ const VIEW_GRAPH_SENTINEL = '?__dkgViewGraph'; * nowhere to hide the sentinel from a `SELECT *` projection). */ function wrapWithGraphValues(sparql: string, graphUris: string[]): string | null { - if (hasGraphClause(sparql)) return sparql; - if (graphUris.length === 0) return sparql; - - const braceStart = findWhereBraceStart(sparql); - if (braceStart === -1) return null; - const braceEnd = findMatchingCloseBrace(sparql, braceStart); - if (braceEnd === -1) return null; - - const before = sparql.slice(0, braceStart + 1); - const inner = sparql.slice(braceStart + 1, braceEnd); - const after = sparql.slice(braceEnd); - - if (graphUris.length === 1) { - return `${before} GRAPH <${assertSafeIri(graphUris[0])}> { ${inner} } ${after}`; - } - - // An inner top-level UNION stays on the per-graph fallback (#789): merging - // that shape across graphs is form-aware there, and this keeps that path and - // its tests unchanged. Same guard `wrapWithGraphUnion` uses. - if (/\bUNION\b/i.test(inner)) return null; - - // Never clamp a variable the user actually uses — fall back instead. - const sentinelName = VIEW_GRAPH_SENTINEL.slice(1); - if (collectQueryVariables(sparql).some((v) => v.slice(1) === sentinelName)) { - return null; - } + return wrapWithProjectedGraphSubselect( + sparql, + graphUris, + [VIEW_GRAPH_SENTINEL], + (inner, graphs) => { + const values = graphs.map((graph) => `<${assertSafeIri(graph)}>`).join(' '); + return `VALUES ${VIEW_GRAPH_SENTINEL} { ${values} } GRAPH ${VIEW_GRAPH_SENTINEL} { ${inner} }`; + }, + ); +} - const innerVars = collectQueryVariables(inner); - if (innerVars.length === 0) { - // Var-less WHERE body (all-constant triples, e.g. `SELECT * WHERE { - // }`). We cannot hide the sentinel here: a sub-SELECT projecting - // the empty user-variable set is not legal SPARQL, and the bare form would - // let `SELECT *` project the injected `?__dkgViewGraph` (leaking the SWM - // graph IRI, which embeds a wallet address). Decline so the caller falls - // back to the union / per-graph path, which binds no graph variable. Such - // a query over many graphs is not a real view workload, so the fallback's - // cost is irrelevant. - return null; +/** + * The mirror anti-join is valid only for a flat basic graph pattern (plus + * FILTER expressions). Nested graph patterns can differ by boundness, where a + * correlated NOT EXISTS compatibility check is not exact mapping equality. + */ +function isDedupSafeBasicGraphPattern(inner: string): boolean { + const forbidden = ['OPTIONAL', 'MINUS', 'SERVICE', 'VALUES', 'BIND', 'SELECT', 'GRAPH', 'EXISTS']; + let i = 0; + while (i < inner.length) { + const ch = inner[i]; + if (ch === '#') { + while (i < inner.length && inner[i] !== '\n') i++; + continue; + } + if (ch === '"' || ch === "'") { + i = skipSparqlStringLiteral(inner, i); + continue; + } + if (ch === '<') { + const end = skipSparqlIriRef(inner, i); + i = end ?? i + 1; + continue; + } + if (ch === '{' || ch === '}') return false; + if (isKeywordStart(inner, i)) { + let end = i + 1; + while (end < inner.length && isWordContinuation(inner[end])) end++; + if (forbidden.some((keyword) => isSparqlKeyword(inner, i, end, keyword))) { + return false; + } + i = end; + continue; + } + i++; } - - const values = graphUris.map((g) => `<${assertSafeIri(g)}>`).join(' '); - const graphBlock = `VALUES ${VIEW_GRAPH_SENTINEL} { ${values} } GRAPH ${VIEW_GRAPH_SENTINEL} { ${inner} }`; - // Hide the sentinel behind a sub-SELECT that re-exposes only the user's - // variables, so `SELECT *` and cross-graph DISTINCT behave as they did under - // the UNION form. - return `${before} { SELECT ${innerVars.join(' ')} WHERE { ${graphBlock} } } ${after}`; + return true; } /** diff --git a/packages/query/test/read-both-dedup.test.ts b/packages/query/test/read-both-dedup.test.ts new file mode 100644 index 0000000000..3707913593 --- /dev/null +++ b/packages/query/test/read-both-dedup.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; +import { OxigraphStore, type Quad } from '@origintrail-official/dkg-storage'; +import { DKGQueryEngine } from '../src/dkg-query-engine.js'; + +const CG = 'finalization-chain-e2e'; +const ROOT = `did:dkg:context-graph:${CG}`; +const PER_CGID = `${ROOT}/context/7`; +const VM = `${ROOT}/_verifiable_memory/0xAA/1`; +const ENTITY = 'urn:finalization-chain:entity:1'; +const NAME = 'http://schema.org/name'; +const TYPE = 'http://schema.org/additionalType'; + +function q(subject: string, predicate: string, object: string, graph: string): Quad { + return { subject, predicate, object, graph }; +} + +const NAME_QUERY = `SELECT ?name WHERE { <${ENTITY}> <${NAME}> ?name }`; + +describe('verifiable-memory read-both deduplicates mirrored triples (#1270)', () => { + it('collapses an identical triple mirrored between root and per-KA VM', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const triple = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([triple, { ...triple, graph: VM }]); + + const result = await engine.query(NAME_QUERY, { contextGraphId: CG }); + + expect(result.bindings).toEqual([{ name: '"Finalization Chain Draft"' }]); + }); + + it('preserves SELECT bag multiplicity for distinct triples with the same projection', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const name = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([ + name, + q(ENTITY, TYPE, '"Document"', ROOT), + { ...name, graph: VM }, + ]); + + const result = await engine.query( + `SELECT ?s WHERE { ?s ?p ?o . FILTER(?s = <${ENTITY}>) }`, + { contextGraphId: CG }, + ); + + expect(result.bindings).toEqual([{ s: ENTITY }, { s: ENTITY }]); + }); + + it('preserves mappings that differ by an OPTIONAL binding in a later VM graph', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const name = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([ + name, + { ...name, graph: VM }, + q(ENTITY, TYPE, '"Document"', VM), + ]); + + const result = await engine.query( + `SELECT ?name ?type WHERE { + <${ENTITY}> <${NAME}> ?name . + OPTIONAL { <${ENTITY}> <${TYPE}> ?type } + }`, + { contextGraphId: CG, view: 'verifiable-memory' }, + ); + + expect(result.bindings).toHaveLength(2); + expect(result.bindings.filter((binding) => binding['type'] === undefined)).toHaveLength(1); + expect(result.bindings.filter((binding) => binding['type'] === '"Document"')).toHaveLength(1); + }); + + it('actually reads and deduplicates the per-cgId graph in VM view routing', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const triple = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([triple, { ...triple, graph: PER_CGID }]); + + const result = await engine.query(NAME_QUERY, { + contextGraphId: CG, + view: 'verifiable-memory', + }); + + expect(result.bindings).toEqual([{ name: '"Finalization Chain Draft"' }]); + }); + + it('deduplicates a per-cgId and per-KA mirror when the root graph is empty', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const triple = q(ENTITY, NAME, '"Finalization Chain Draft"', PER_CGID); + await store.insert([triple, { ...triple, graph: VM }]); + + const result = await engine.query(NAME_QUERY, { + contextGraphId: CG, + view: 'verifiable-memory', + }); + + expect(result.bindings).toEqual([{ name: '"Finalization Chain Draft"' }]); + }); +}); diff --git a/packages/rdf-utils/README.md b/packages/rdf-utils/README.md new file mode 100644 index 0000000000..b5f5fd802d --- /dev/null +++ b/packages/rdf-utils/README.md @@ -0,0 +1,4 @@ +# DKG RDF utilities + +Dependency-free RDF serialization helpers shared by DKG packages that must not +depend on the full DKG core runtime. diff --git a/packages/rdf-utils/package.json b/packages/rdf-utils/package.json new file mode 100644 index 0000000000..3fb8e4cdb1 --- /dev/null +++ b/packages/rdf-utils/package.json @@ -0,0 +1,38 @@ +{ + "name": "@origintrail-official/dkg-rdf-utils", + "version": "10.0.6", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "scripts": { + "build": "tsc", + "test": "vitest run", + "test:coverage": "vitest run --coverage", + "clean": "rm -rf dist tsconfig.tsbuildinfo" + }, + "devDependencies": { + "@vitest/coverage-v8": "^4.0.18", + "vitest": "^4.0.18" + }, + "publishConfig": { + "access": "public" + }, + "files": [ + "dist", + "README.md" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/OriginTrail/dkg.git", + "directory": "packages/rdf-utils" + } +} diff --git a/packages/rdf-utils/src/index.ts b/packages/rdf-utils/src/index.ts new file mode 100644 index 0000000000..9b0a8cc0a0 --- /dev/null +++ b/packages/rdf-utils/src/index.ts @@ -0,0 +1,39 @@ +/** N-Triples ECHAR short forms, keyed by the raw character. */ +const RDF_LITERAL_SHORT_ESCAPES: Readonly> = Object.freeze({ + '\b': '\\b', + '\t': '\\t', + '\n': '\\n', + '\f': '\\f', + '\r': '\\r', + '"': '\\"', + '\\': '\\\\', +}); + +const RDF_LITERAL_ESCAPE_PATTERN = /["\\\u0000-\u001F\u007F]/g; + +/** + * Escape a plain-text string for use as an RDF/N-Triples literal body. + * Returns only the escaped body; callers add the surrounding quotes. + */ +export function escapeRdfLiteral(value: string): string { + return value.replace(RDF_LITERAL_ESCAPE_PATTERN, (character) => { + const shortEscape = RDF_LITERAL_SHORT_ESCAPES[character]; + if (shortEscape !== undefined) return shortEscape; + return `\\u${character.charCodeAt(0).toString(16).toUpperCase().padStart(4, '0')}`; + }); +} + +/** Return whether a string already represents an RDF term accepted by DKG publishers. */ +export function isRdfTerm(value: string): boolean { + return ( + /^(?:https?:\/\/|urn:|did:)/i.test(value) || + value.startsWith('_:') || + value.startsWith('"') + ); +} + +/** Preserve RDF terms and quote/escape every other value as a plain literal. */ +export function normalizeRdfObject(value: unknown): string { + const raw = String(value ?? ''); + return isRdfTerm(raw) ? raw : `"${escapeRdfLiteral(raw)}"`; +} diff --git a/packages/rdf-utils/test/rdf-literal.test.ts b/packages/rdf-utils/test/rdf-literal.test.ts new file mode 100644 index 0000000000..bbc5ee637a --- /dev/null +++ b/packages/rdf-utils/test/rdf-literal.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { escapeRdfLiteral, isRdfTerm, normalizeRdfObject } from '../src/index.js'; + +describe('escapeRdfLiteral', () => { + it('escapes quotes, backslashes, ECHAR controls, remaining C0 controls, and DEL', () => { + expect(escapeRdfLiteral('q"\\b\bt\tn\nf\fr\rnul\u0000vt\u000Bus\u001Fdel\u007F')).toBe( + 'q\\"\\\\b\\bt\\tn\\nf\\fr\\rnul\\u0000vt\\u000Bus\\u001Fdel\\u007F', + ); + }); + + it('owns the complete dependency-free RDF object normalization boundary', () => { + expect(isRdfTerm('urn:test:entity')).toBe(true); + expect(isRdfTerm('_:blank')).toBe(true); + expect(isRdfTerm('plain')).toBe(false); + expect(normalizeRdfObject('urn:test:entity')).toBe('urn:test:entity'); + expect(normalizeRdfObject('a "quote"\u0000')).toBe('"a \\"quote\\"\\u0000"'); + expect(normalizeRdfObject(null)).toBe('""'); + }); +}); diff --git a/packages/rdf-utils/tsconfig.json b/packages/rdf-utils/tsconfig.json new file mode 100644 index 0000000000..d231bbc57e --- /dev/null +++ b/packages/rdf-utils/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "composite": true + }, + "include": ["src"] +} diff --git a/packages/rdf-utils/vitest.config.ts b/packages/rdf-utils/vitest.config.ts new file mode 100644 index 0000000000..43e56f45f3 --- /dev/null +++ b/packages/rdf-utils/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + }, +}); diff --git a/packages/storage/src/graph-manager.ts b/packages/storage/src/graph-manager.ts index 1e8ae690a6..87a7edbe9b 100644 --- a/packages/storage/src/graph-manager.ts +++ b/packages/storage/src/graph-manager.ts @@ -47,6 +47,17 @@ export interface SwmKaGraphBound { endNumber: bigint; } +/** Exact identity of one named KA lifecycle's shared-memory graph. */ +export interface NamedKnowledgeAssetGraphIdentity { + agentAddress: string; + kaNumber: bigint; +} + +/** Semantic SWM read boundary: either the complete family or one named lifecycle. */ +export type SharedMemoryGraphScope = + | { kind: 'complete-family' } + | { kind: 'named-lifecycle'; identity: NamedKnowledgeAssetGraphIdentity }; + const SWM_CHILD_AGENT_ADDRESS = /^0x[0-9a-fA-F]{40}$/; const SWM_CHILD_KA_NUMBER = /^\d+$/; @@ -146,9 +157,10 @@ async function listGraphsByPrefix( * paths (recompute mismatch → reject/retry, never accept-with-wrong-data). * * This resolver is COMPLETE and therefore safe everywhere, including the - * merkle-defining publish reads and the StorageACK decline lanes. Pruning lives in - * `resolveKaBoundedSharedMemoryReadGraphs`, which is not part of the package's - * public surface — read its contract before reaching for it. + * merkle-defining publish reads and the StorageACK decline lanes. Generic pruning + * lives in `resolveKaBoundedSharedMemoryReadGraphs`, which is not part of the + * package's public surface. The public exact-named-lifecycle API below is a + * separate semantic boundary, not a range-pruning escape hatch. */ export async function resolveSharedMemoryReadGraphs( store: TripleStore, @@ -230,13 +242,22 @@ export async function loadSelectedSharedMemoryQuads( selection: SharedMemoryReadSelection, options: LoadSelectedSharedMemoryQuadsOptions = {}, ): Promise { - return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, undefined); + return loadSharedMemoryQuadsForScope( + store, + bucketGraph, + selection, + { kind: 'complete-family' }, + options, + ); } /** * Load the SWM quad slice pruned to ONE author's per-KA under-graphs (#1549). * - * UNSAFE ON ITS OWN, and deliberately NOT re-exported from `src/index.ts`. The + * UNSAFE as a generic merkle accelerator. It is module-exported for direct + * tests and internal imports, but is not re-exported from the package entrypoint. + * Exact lifecycle callers use the scoped public loader below. Generic merkle + * callers must use the widening wrapper below. The * pruned graph set is a strict subset of the set `loadSelectedSharedMemoryQuads` * reads, and INV-1 — "a root's quads live only under its own KA number" — is * REFUTED under root recurrence, so this read can legitimately miss quads the @@ -252,7 +273,97 @@ export async function loadKaBoundedSharedMemoryQuads( kaGraphBound: SwmKaGraphBound, options: LoadSelectedSharedMemoryQuadsOptions = {}, ): Promise { - return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, kaGraphBound); + return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, { + kind: 'bounded', + bound: kaGraphBound, + }); +} + +/** + * Load shared memory through one explicit semantic scope. + * + * Higher layers do not translate scope into concrete graph policy: complete + * family and exact named-lifecycle dispatch both stay owned by storage. + */ +export async function loadSharedMemoryQuadsForScope( + store: TripleStore, + bucketGraph: string, + selection: SharedMemoryReadSelection, + scope: SharedMemoryGraphScope, + options: LoadSelectedSharedMemoryQuadsOptions = {}, +): Promise { + return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, scope); +} + +interface NamedLifecycleGraphResolution { + canonicalGraph: string; + matchingGraphs: string[]; +} + +async function resolveNamedLifecycleGraphPolicy( + store: TripleStore, + bucketGraph: string, + identity: NamedKnowledgeAssetGraphIdentity, + options?: QueryOptions, +): Promise { + assertSafeIri(bucketGraph); + const { agentAddress, kaNumber } = identity; + if (!SWM_CHILD_AGENT_ADDRESS.test(agentAddress) || kaNumber < 0n) { + throw new Error('Named KA graph identity must contain a 20-byte EVM address and non-negative KA number'); + } + const canonicalGraph = `${bucketGraph}/${agentAddress}/${kaNumber.toString()}`; + assertSafeIri(canonicalGraph); + const matchingGraphs = (await listGraphsByPrefix(store, `${bucketGraph}/`, options)) + .filter((graph) => { + const child = parseBoundableSwmChildGraph(bucketGraph, graph); + return child?.agentAddress.toLowerCase() === agentAddress.toLowerCase() + && child.kaNumber === kaNumber; + }); + return { canonicalGraph, matchingGraphs }; +} + +/** Resolve the concrete graph set for an explicit semantic SWM scope. */ +export async function resolveSharedMemoryScopeGraphs( + store: TripleStore, + bucketGraph: string, + scope: SharedMemoryGraphScope, + options?: QueryOptions, +): Promise { + if (scope.kind === 'complete-family') { + return resolveSharedMemoryReadGraphs(store, bucketGraph, options); + } + const { canonicalGraph, matchingGraphs } = await resolveNamedLifecycleGraphPolicy( + store, + bucketGraph, + scope.identity, + options, + ); + // Preserve the writer's checksum casing while matching EVM identity + // case-insensitively. An absent lifecycle still resolves to its canonical + // candidate so the caller gets a safe empty result. + return matchingGraphs.length > 0 + ? matchingGraphs as NonEmptyGraphList + : [canonicalGraph]; +} + +/** Resolve the single graph to WRITE for a semantic scope. */ +export async function resolveSharedMemoryScopeWriteGraph( + store: TripleStore, + bucketGraph: string, + scope: SharedMemoryGraphScope, + options?: QueryOptions, +): Promise { + assertSafeIri(bucketGraph); + if (scope.kind === 'complete-family') return bucketGraph; + const { canonicalGraph, matchingGraphs } = await resolveNamedLifecycleGraphPolicy( + store, + bucketGraph, + scope.identity, + options, + ); + return matchingGraphs.find((graph) => graph === canonicalGraph) + ?? matchingGraphs.slice().sort()[0] + ?? canonicalGraph; } /** Query-source tags for the three read lanes a bounded slice can take. */ @@ -320,7 +431,9 @@ async function loadSharedMemoryQuadsInternal( bucketGraph: string, selection: SharedMemoryReadSelection, options: LoadSelectedSharedMemoryQuadsOptions, - kaGraphBound: SwmKaGraphBound | undefined, + graphScope: + | { kind: 'bounded'; bound: SwmKaGraphBound } + | SharedMemoryGraphScope, ): Promise { let innerGraphPattern: string; if (selection === 'all') { @@ -353,9 +466,22 @@ async function loadSharedMemoryQuadsInternal( } const queryOptions = mergeQueryOptions(options.queryOptions, options.querySource); - const swmGraphs = kaGraphBound - ? await resolveKaBoundedSharedMemoryReadGraphs(store, bucketGraph, kaGraphBound, queryOptions) - : await resolveSharedMemoryReadGraphs(store, bucketGraph, queryOptions); + let swmGraphs: NonEmptyGraphList; + if (graphScope?.kind === 'bounded') { + swmGraphs = await resolveKaBoundedSharedMemoryReadGraphs( + store, + bucketGraph, + graphScope.bound, + queryOptions, + ); + } else { + swmGraphs = await resolveSharedMemoryScopeGraphs( + store, + bucketGraph, + graphScope, + queryOptions, + ); + } const graphValues = swmGraphs.map((g) => `<${g}>`).join(' '); const result = await store.query(`CONSTRUCT { ?s ?p ?o } WHERE { VALUES ?g { ${graphValues} } diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 5e8f4ee97c..9bc3637e82 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -72,14 +72,19 @@ export { ContextGraphManager, GraphManager, loadSelectedSharedMemoryQuads, + loadSharedMemoryQuadsForScope, loadSharedMemorySliceWithKaBoundFallback, + resolveSharedMemoryScopeGraphs, + resolveSharedMemoryScopeWriteGraph, loadSelectedVerifiableMemoryQuads, resolveSharedMemoryReadGraphs, resolveVerifiableMemoryReadGraphs, type LoadSelectedSharedMemoryQuadsOptions, type LoadSelectedVerifiableMemoryQuadsOptions, type NonEmptyGraphList, + type NamedKnowledgeAssetGraphIdentity, type SharedMemoryReadSelection, + type SharedMemoryGraphScope, type SwmKaGraphBound, type SwmSliceSourceTags, } from './graph-manager.js'; diff --git a/packages/storage/test/graph-manager-swm-bound.test.ts b/packages/storage/test/graph-manager-swm-bound.test.ts index 61af795ed3..7e3d512dc7 100644 --- a/packages/storage/test/graph-manager-swm-bound.test.ts +++ b/packages/storage/test/graph-manager-swm-bound.test.ts @@ -2,8 +2,10 @@ import { describe, it, expect } from 'vitest'; import * as storageIndex from '../src/index.js'; import { createTripleStore, + loadSharedMemoryQuadsForScope, loadSelectedSharedMemoryQuads, loadSharedMemorySliceWithKaBoundFallback, + resolveSharedMemoryScopeWriteGraph, resolveSharedMemoryReadGraphs, type Quad, type SwmKaGraphBound, @@ -234,6 +236,36 @@ describe('resolveSharedMemoryReadGraphs — bound only prunes real SWM children }); describe('the generic SWM loader cannot be pruned (bound is not an option)', () => { + it('exact named-KA reads preserve checksum graph casing and exclude the bucket', async () => { + const store = await createTripleStore({ backend: 'oxigraph' }); + const swm = contextGraphSharedMemoryUri('named-exact-casing'); + const root = 'urn:test:named:root'; + const exact = `${swm}/${AUTHOR_A_MIXED}/7`; + const sameAuthorSibling = `${swm}/${AUTHOR_A_MIXED}/8`; + try { + await store.insert([ + { subject: root, predicate: 'urn:p', object: '"bucket"', graph: swm }, + { subject: root, predicate: 'urn:p', object: '"exact"', graph: exact }, + { subject: root, predicate: 'urn:p', object: '"same-author-sibling"', graph: sameAuthorSibling }, + ]); + + const scope = { + kind: 'named-lifecycle', + identity: { agentAddress: AUTHOR_A, kaNumber: 7n }, + } as const; + const quads = await loadSharedMemoryQuadsForScope( + store, + swm, + { rootEntities: [root] }, + scope, + ); + expect(quads.map((quad) => quad.object)).toEqual(['"exact"']); + expect(await resolveSharedMemoryScopeWriteGraph(store, swm, scope)).toBe(exact); + } finally { + await store.close(); + } + }); + // `kaGraphBound` was removed from `LoadSelectedSharedMemoryQuadsOptions`, so the // four production callers — two of them merkle-DEFINING, one the ACK decline lane // — get a compile error if they try to prune. This pins the runtime half: even if @@ -297,6 +329,10 @@ describe('the generic SWM loader cannot be pruned (bound is not an option)', () expect(storageIndex).not.toHaveProperty('resolveKaBoundedSharedMemoryReadGraphs'); // The safe, fallback-owning primitive IS public. expect(typeof storageIndex.loadSharedMemorySliceWithKaBoundFallback).toBe('function'); + // Named publish flows get a scoped API, not a second range-shaped loader. + expect(typeof storageIndex.loadSharedMemoryQuadsForScope).toBe('function'); + expect(typeof storageIndex.resolveSharedMemoryScopeWriteGraph).toBe('function'); + expect(storageIndex).not.toHaveProperty('loadNamedKnowledgeAssetSharedMemoryQuads'); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d31b0a2e05..d5e09d3592 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -654,6 +654,9 @@ importers: '@opentelemetry/api': specifier: ^1.9.1 version: 1.9.1 + '@origintrail-official/dkg-rdf-utils': + specifier: workspace:* + version: link:../rdf-utils js-yaml: specifier: ^4.1.1 version: 4.1.1 @@ -846,6 +849,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1 version: 1.27.1(zod@3.25.76) + '@origintrail-official/dkg-rdf-utils': + specifier: workspace:* + version: link:../rdf-utils yaml: specifier: ^2.6.0 version: 2.8.3 @@ -1109,6 +1115,15 @@ importers: specifier: ^4.0.18 version: 4.0.18(@opentelemetry/api@1.9.1)(@types/node@22.19.11)(happy-dom@20.8.9(bufferutil@4.1.0)(utf-8-validate@5.0.10))(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) + packages/rdf-utils: + devDependencies: + '@vitest/coverage-v8': + specifier: ^4.0.18 + version: 4.0.18(vitest@4.0.18(@opentelemetry/api@1.9.1)(@types/node@22.19.11)(happy-dom@20.8.9(bufferutil@4.1.0)(utf-8-validate@5.0.10))(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) + vitest: + specifier: ^4.0.18 + version: 4.0.18(@opentelemetry/api@1.9.1)(@types/node@22.19.11)(happy-dom@20.8.9(bufferutil@4.1.0)(utf-8-validate@5.0.10))(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) + packages/storage: dependencies: '@origintrail-official/dkg-core': diff --git a/scripts/devnet-test-issue-1561-rpc-throttle-retry.sh b/scripts/devnet-test-issue-1561-rpc-throttle-retry.sh new file mode 100755 index 0000000000..ff4f5f58fb --- /dev/null +++ b/scripts/devnet-test-issue-1561-rpc-throttle-retry.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1561. A temporary node uses two logical RPC +# endpoints behind a fault proxy. The first eth_getBlockByNumber on each endpoint +# returns 429 (one complete throttled pool); the next pool pass forwards and the +# publish must confirm. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +UPSTREAM="${DEVNET_RPC:-http://127.0.0.1:8545}" +PROXY_PORT="${THROTTLE_PROXY_PORT:-18561}" +NODE="${THROTTLE_TEST_NODE:-7}" +NODE_DIR="$DEVNET_DIR/node$NODE" +CG="${DEVNET_CONTEXT_GRAPH:-devnet-test}" +proxy_pid='' + +fail() { echo "[#1561] FAIL: $*" >&2; exit 1; } +cleanup() { + "$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null 2>&1 || true + rm -rf "$NODE_DIR" + [[ -n "$proxy_pid" ]] && kill "$proxy_pid" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM +[[ ! -d "$NODE_DIR" ]] || fail "$NODE_DIR already exists" + +UPSTREAM="$UPSTREAM" PROXY_PORT="$PROXY_PORT" node --input-type=module <<'NODE' & +import http from 'node:http'; +const counts = new Map(); +http.createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/stats') { + res.setHeader('content-type', 'application/json'); + return res.end(JSON.stringify(Object.fromEntries(counts))); + } + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = Buffer.concat(chunks); + let method = ''; + try { method = JSON.parse(body.toString()).method; } catch {} + const key = `${req.url}:${method}`; + const count = counts.get(key) ?? 0; + counts.set(key, count + 1); + if (method === 'eth_getBlockByNumber' && count === 0) { + res.statusCode = 429; + res.setHeader('content-type', 'application/json'); + return res.end(JSON.stringify({ error: 'Too Many Requests' })); + } + const upstream = await fetch(process.env.UPSTREAM, { + method: 'POST', headers: { 'content-type': 'application/json' }, body, + }); + res.statusCode = upstream.status; + res.end(Buffer.from(await upstream.arrayBuffer())); +}).listen(Number(process.env.PROXY_PORT), '127.0.0.1'); +NODE +proxy_pid=$! +sleep 1 +curl -fsS "http://127.0.0.1:$PROXY_PORT/stats" >/dev/null || fail "fault proxy did not start" + +"$ROOT/scripts/devnet.sh" addnode "$NODE" edge >/dev/null +"$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null +NODE_DIR="$NODE_DIR" PROXY_PORT="$PROXY_PORT" node --input-type=module <<'NODE' +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +const path = join(process.env.NODE_DIR, 'config.json'); +const config = JSON.parse(readFileSync(path, 'utf8')); +const base = `http://127.0.0.1:${process.env.PROXY_PORT}`; +config.chain.rpcUrl = `${base}/a`; +config.chain.rpcUrls = [`${base}/a`, `${base}/b`]; +writeFileSync(path, JSON.stringify(config, null, 2)); +NODE +"$ROOT/scripts/devnet.sh" restart-node "$NODE" >/dev/null + +. "$ROOT/scripts/devnet-lib.sh" +for _ in $(seq 1 90); do + [[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] && break + sleep 1 +done +[[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] || fail "temporary node not ready" +api "$NODE" POST /api/identity/ensure '{}' >/dev/null || true + +name="issue-1561-$(date +%s)-$$"; subject="urn:issue:1561:$name" +api "$NODE" POST /api/knowledge-assets "{\"contextGraphId\":\"$CG\",\"name\":\"$name\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/write" \ + "{\"contextGraphId\":\"$CG\",\"quads\":[{\"subject\":\"$subject\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"429 recovery probe\\\"\",\"graph\":\"\"}]}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/finalize" "{\"contextGraphId\":\"$CG\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/swm/share" "{\"contextGraphId\":\"$CG\"}" >/dev/null +result="$(api "$NODE" POST "/api/knowledge-assets/$name/vm/publish" "{\"contextGraphId\":\"$CG\"}")" +[[ "$(code_of "$result")" == 200 ]] || fail "publish failed: $(body_of "$result")" +[[ "$(field "$(body_of "$result")" status)" == confirmed ]] || fail "publish not confirmed" + +stats="$(curl -fsS "http://127.0.0.1:$PROXY_PORT/stats")" +STATS="$stats" node -e 'const s=JSON.parse(process.env.STATS); for (const p of ["/a","/b"]) { const n=s[`${p}:eth_getBlockByNumber`]||0; if(n<2) throw new Error(`${p} getBlock calls=${n}, expected throttle plus recovery`); }' \ + || fail "proxy did not observe a full throttled pass plus recovery: $stats" +echo "[#1561] PASS: publish recovered after both RPC endpoints returned 429" diff --git a/scripts/devnet-test-issue-1567-global-cli-rollback.sh b/scripts/devnet-test-issue-1567-global-cli-rollback.sh new file mode 100755 index 0000000000..f35aa8e708 --- /dev/null +++ b/scripts/devnet-test-issue-1567-global-cli-rollback.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Host-level devnet regression for #1567. It exercises the real edge updater +# orchestration in an isolated npm prefix/PATH: the target install reports a +# semver-prefix collision (rc.12 for expected rc.1), so verification must reject +# it and the previous rc.0 CLI must be restored. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +tmp="$(mktemp -d "${TMPDIR:-/tmp}/dkg-1567.XXXXXX")" +trap 'rm -rf "$tmp"' EXIT INT TERM +mkdir -p "$tmp/bin" "$tmp/home" +version_file="$tmp/version" +printf '%s\n' '10.0.0-rc.0' >"$version_file" + +printf '%s\n' '#!/bin/sh' \ + 'case "$*" in' \ + ' *10.0.0-rc.1*) printf "%s\n" "10.0.0-rc.12" >"$FAKE_DKG_VERSION_FILE" ;;' \ + ' *10.0.0-rc.0*) printf "%s\n" "10.0.0-rc.0" >"$FAKE_DKG_VERSION_FILE" ;;' \ + ' *) exit 2 ;;' \ + 'esac' >"$tmp/bin/npm" +printf '%s\n' '#!/bin/sh' \ + 'printf "dkg %s\n" "$(cat "$FAKE_DKG_VERSION_FILE")"' >"$tmp/bin/dkg" +chmod +x "$tmp/bin/npm" "$tmp/bin/dkg" + +result="$( + PATH="$tmp/bin:$PATH" DKG_HOME="$tmp/home" FAKE_DKG_VERSION_FILE="$version_file" \ + node --input-type=module <<'NODE' +import { performNpmUpdateEdge } from './packages/cli/dist/daemon/auto-update.js'; +const logs = []; +const result = await performNpmUpdateEdge('10.0.0-rc.1', '10.0.0-rc.0', (line) => logs.push(line)); +process.stdout.write(JSON.stringify({ result, logs })); +NODE +)" + +RESULT="$result" node -e ' +const value = JSON.parse(process.env.RESULT); +if (value.result !== "failed") throw new Error(`expected failed-with-rollback, got ${value.result}`); +if (!value.logs.some((line) => line.includes("expected 10.0.0-rc.1"))) throw new Error("exact mismatch was not detected"); +if (!value.logs.some((line) => line.includes("rollback restored"))) throw new Error("rollback was not verified"); +' +[[ "$(cat "$version_file")" == 10.0.0-rc.0 ]] || { echo "[#1567] FAIL: previous CLI not restored" >&2; exit 1; } +[[ "$(cat "$tmp/home/previous-version")" == 10.0.0-rc.0 ]] || { echo "[#1567] FAIL: rollback target not recorded" >&2; exit 1; } +echo "[#1567] PASS: exact-version self-check rejected rc.12 and restored rc.0" diff --git a/scripts/devnet-test-issue-1576-publisher-readiness.sh b/scripts/devnet-test-issue-1576-publisher-readiness.sh new file mode 100755 index 0000000000..5431b4e6e4 --- /dev/null +++ b/scripts/devnet-test-issue-1576-publisher-readiness.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1576. A temporary edge node is started with +# publisher.enabled=true but no publisher wallet. publish-async must return 503 +# before persistence, and the durable publisher job count must not change. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +NODE="${PUBLISHER_READINESS_NODE:-7}" +NODE_DIR="$DEVNET_DIR/node$NODE" +CG="${DEVNET_CONTEXT_GRAPH:-devnet-test}" + +fail() { echo "[#1576] FAIL: $*" >&2; exit 1; } +cleanup() { + "$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null 2>&1 || true + rm -rf "$NODE_DIR" +} +trap cleanup EXIT INT TERM + +[[ -d "$DEVNET_DIR/node1" ]] || fail "start the baseline devnet first" +[[ ! -d "$NODE_DIR" ]] || fail "$NODE_DIR already exists; choose PUBLISHER_READINESS_NODE" +"$ROOT/scripts/devnet.sh" addnode "$NODE" edge >/dev/null +"$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null + +NODE_DIR="$NODE_DIR" node --input-type=module <<'NODE' +import { readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +const path = join(process.env.NODE_DIR, 'config.json'); +const config = JSON.parse(readFileSync(path, 'utf8')); +config.publisher = { ...(config.publisher ?? {}), enabled: true }; +writeFileSync(path, JSON.stringify(config, null, 2)); +rmSync(join(process.env.NODE_DIR, 'publisher-wallets.json'), { force: true }); +NODE +"$ROOT/scripts/devnet.sh" restart-node "$NODE" >/dev/null + +. "$ROOT/scripts/devnet-lib.sh" +for _ in $(seq 1 90); do + [[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] && break + sleep 1 +done +[[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] || fail "temporary node did not become ready" +api "$NODE" POST /api/identity/ensure '{}' >/dev/null || true + +jobs_before_body="$(body_of "$(api "$NODE" GET /api/publisher/jobs)")" +jobs_before="$(JOBS="$jobs_before_body" node -e 'const j=JSON.parse(process.env.JOBS);process.stdout.write(String((j.jobs||[]).length))')" + +name="issue-1576-$(date +%s)-$$" +subject="urn:issue:1576:$name" +api "$NODE" POST /api/knowledge-assets "{\"contextGraphId\":\"$CG\",\"name\":\"$name\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/write" \ + "{\"contextGraphId\":\"$CG\",\"quads\":[{\"subject\":\"$subject\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"publisher readiness probe\\\"\",\"graph\":\"\"}]}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/finalize" "{\"contextGraphId\":\"$CG\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/swm/share" "{\"contextGraphId\":\"$CG\"}" >/dev/null + +response="$(api "$NODE" POST "/api/knowledge-assets/$name/vm/publish-async" "{\"contextGraphId\":\"$CG\"}")" +[[ "$(code_of "$response")" == 503 ]] || fail "expected 503, got $(code_of "$response"): $(body_of "$response")" +body="$(body_of "$response")" +[[ "$(field "$body" code)" == async_publisher_unavailable ]] || fail "wrong stable error code: $body" +[[ "$(field "$body" reason)" == no_publisher_wallets ]] || fail "wrong unavailable reason: $body" +[[ "$(field "$body" retryable)" == false ]] || fail "no-wallet state was advertised retryable" + +jobs_after_body="$(body_of "$(api "$NODE" GET /api/publisher/jobs)")" +jobs_after="$(JOBS="$jobs_after_body" node -e 'const j=JSON.parse(process.env.JOBS);process.stdout.write(String((j.jobs||[]).length))')" +[[ "$jobs_after" == "$jobs_before" ]] || fail "job count grew from $jobs_before to $jobs_after" +echo "[#1576] PASS: unavailable publisher rejected before durable enqueue" diff --git a/scripts/devnet-test-issue-1577-ack-outbox-ownership.sh b/scripts/devnet-test-issue-1577-ack-outbox-ownership.sh new file mode 100755 index 0000000000..240004ea87 --- /dev/null +++ b/scripts/devnet-test-issue-1577-ack-outbox-ownership.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1577. Stop one of four cores, publish from an edge +# (the other three can still satisfy quorum), then prove the finished collector +# left no durable StorageACK request behind on the publisher. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +PUBLISHER="${ACK_OUTBOX_PUBLISHER:-5}" +TARGET="${ACK_OUTBOX_TARGET:-4}" +CG="${DEVNET_CONTEXT_GRAPH:-devnet-test}" +DB="$DEVNET_DIR/node$PUBLISHER/node-ui.db" +stopped=0 + +fail() { echo "[#1577] FAIL: $*" >&2; exit 1; } +cleanup() { + if [[ "$stopped" == 1 ]]; then "$ROOT/scripts/devnet.sh" restart-node "$TARGET" >/dev/null 2>&1 || true; fi +} +trap cleanup EXIT INT TERM +. "$ROOT/scripts/devnet-lib.sh" + +for n in 1 2 3 4 5; do + [[ "$(code_of "$(api "$n" GET /api/status)")" == 200 ]] || fail "node$n is not ready (need 4 cores + edge)" +done +[[ -f "$DB" ]] || fail "publisher database missing: $DB" + +count_ack_rows() { + DB="$DB" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare(`SELECT COUNT(*) AS n FROM protocol_outbox + WHERE protocol IN ('/dkg/10.0.1/storage-ack','/dkg/10.0.2/storage-ack','/dkg/10.0.1/storage-update-ack')`).get(); +process.stdout.write(String(row.n)); db.close(); +NODE +} +baseline="$(count_ack_rows)" + +"$ROOT/scripts/devnet.sh" stop-node "$TARGET" >/dev/null +stopped=1 +sleep 5 + +name="issue-1577-$(date +%s)-$$" +subject="urn:issue:1577:$name" +api "$PUBLISHER" POST /api/knowledge-assets "{\"contextGraphId\":\"$CG\",\"name\":\"$name\"}" >/dev/null +api "$PUBLISHER" POST "/api/knowledge-assets/$name/wm/write" \ + "{\"contextGraphId\":\"$CG\",\"quads\":[{\"subject\":\"$subject\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"ack ownership probe\\\"\",\"graph\":\"\"}]}" >/dev/null +api "$PUBLISHER" POST "/api/knowledge-assets/$name/wm/finalize" "{\"contextGraphId\":\"$CG\"}" >/dev/null +api "$PUBLISHER" POST "/api/knowledge-assets/$name/swm/share" "{\"contextGraphId\":\"$CG\"}" >/dev/null +published="$(api "$PUBLISHER" POST "/api/knowledge-assets/$name/vm/publish" "{\"contextGraphId\":\"$CG\"}")" +[[ "$(code_of "$published")" == 200 ]] || fail "publish did not reach quorum: $(body_of "$published")" +[[ "$(field "$(body_of "$published")" status)" == confirmed ]] || fail "publish was not confirmed" + +rows="$(count_ack_rows)" +[[ "$rows" == "$baseline" ]] || fail "StorageACK outbox rows grew from $baseline to $rows" +echo "[#1577] PASS: collector completed with no durable StorageACK rows" diff --git a/scripts/devnet-test-issue-1579-outbox-backoff.sh b/scripts/devnet-test-issue-1579-outbox-backoff.sh new file mode 100755 index 0000000000..98ec0e1ac4 --- /dev/null +++ b/scripts/devnet-test-issue-1579-outbox-backoff.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1579. A cooling-down reliable message is seeded +# into node1's durable outbox, node2 is restarted/reconnected, and the row must +# remain untouched until its scheduled next_attempt_at. On the buggy build the +# connection:open hook immediately sends/removes (or advances) the row. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +DB="$DEVNET_DIR/node1/node-ui.db" +MESSAGE_ID="devnet-issue-1579-$(date +%s)" + +fail() { echo "[#1579] FAIL: $*" >&2; exit 1; } +cleanup() { + DB="$DB" MESSAGE_ID="$MESSAGE_ID" node --input-type=module <<'NODE' >/dev/null 2>&1 || true +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB); +db.prepare('DELETE FROM protocol_outbox WHERE message_id = ?').run(process.env.MESSAGE_ID); +db.close(); +NODE +} +trap cleanup EXIT + +[[ -f "$DB" ]] || fail "node1 database missing; start a devnet first" +. "$ROOT/scripts/devnet-lib.sh" + +status2="$(body_of "$(api 2 GET /api/status)")" +peer2="$(field "$status2" peerId)" +[[ -n "$peer2" ]] || fail "node2 peerId unavailable" + +DB="$DB" PEER="$peer2" MESSAGE_ID="$MESSAGE_ID" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +import { encodeReliableEnvelope, PROTOCOL_MESSAGE } from './packages/core/dist/index.js'; +const now = Date.now(); +const payload = encodeReliableEnvelope({ + messageId: process.env.MESSAGE_ID, + version: 1, + tsMs: now, + payload: new TextEncoder().encode('{"type":"chat","text":"#1579 probe"}'), +}); +const db = new Database(process.env.DB); +db.prepare(`INSERT INTO protocol_outbox + (peer_id, protocol, message_id, payload, attempts, first_failure_at, + last_attempt_at, next_attempt_at, last_error) + VALUES (?, ?, ?, ?, 1, ?, ?, ?, ?)`) + .run(process.env.PEER, PROTOCOL_MESSAGE, process.env.MESSAGE_ID, + Buffer.from(payload), now, now, now + 60 * 60 * 1000, 'devnet probe'); +db.close(); +NODE + +"$ROOT/scripts/devnet.sh" restart-node 2 >/dev/null +for _ in $(seq 1 60); do + [[ "$(code_of "$(api 2 GET /api/status)")" == 200 ]] && break + sleep 1 +done +[[ "$(code_of "$(api 2 GET /api/status)")" == 200 ]] || fail "node2 did not restart" + +connect="$(api 1 POST /api/connect "{\"peerId\":\"$peer2\"}")" +[[ "$(code_of "$connect")" == 200 ]] || fail "node1 could not reconnect to node2" +sleep 5 + +row="$(DB="$DB" MESSAGE_ID="$MESSAGE_ID" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare('SELECT attempts, next_attempt_at FROM protocol_outbox WHERE message_id = ?').get(process.env.MESSAGE_ID); +process.stdout.write(row ? JSON.stringify(row) : 'missing'); +db.close(); +NODE +)" +[[ "$row" != missing ]] || fail "cooling row was drained on connection-open" +attempts="$(field "$row" attempts)" +[[ "$attempts" == 1 ]] || fail "connection-open advanced attempts to $attempts" +echo "[#1579] PASS: reconnect preserved the cooling-down outbox row" diff --git a/scripts/devnet-test-issue-1580-bounded-outbox-drain.sh b/scripts/devnet-test-issue-1580-bounded-outbox-drain.sh new file mode 100755 index 0000000000..95fe74e4e1 --- /dev/null +++ b/scripts/devnet-test-issue-1580-bounded-outbox-drain.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1580. Seed 250 immediately-due rows and assert one +# scheduler pass consumes/reschedules no more than the configured default batch +# of 100. The unfixed drain loads the full due set in one unbounded pass. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +DB="$DEVNET_DIR/node1/node-ui.db" +PREFIX="devnet-issue-1580-$(date +%s)-" + +fail() { echo "[#1580] FAIL: $*" >&2; exit 1; } +cleanup() { + DB="$DB" PREFIX="$PREFIX" node --input-type=module <<'NODE' >/dev/null 2>&1 || true +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB); +db.prepare('DELETE FROM protocol_outbox WHERE message_id LIKE ?').run(`${process.env.PREFIX}%`); +db.close(); +NODE +} +trap cleanup EXIT +[[ -f "$DB" ]] || fail "node1 database missing; start a devnet first" +. "$ROOT/scripts/devnet-lib.sh" + +status2="$(body_of "$(api 2 GET /api/status)")" +peer2="$(field "$status2" peerId)" +[[ -n "$peer2" ]] || fail "node2 peerId unavailable" + +DB="$DB" PEER="$peer2" PREFIX="$PREFIX" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +import { encodeReliableEnvelope, PROTOCOL_MESSAGE } from './packages/core/dist/index.js'; +const db = new Database(process.env.DB); +const insert = db.prepare(`INSERT INTO protocol_outbox + (peer_id, protocol, message_id, payload, attempts, first_failure_at, + last_attempt_at, next_attempt_at, last_error) + VALUES (?, ?, ?, ?, 1, ?, ?, ?, ?)`); +const now = Date.now(); +const seed = db.transaction(() => { + for (let i = 0; i < 250; i += 1) { + const id = `${process.env.PREFIX}${String(i).padStart(3, '0')}`; + const payload = encodeReliableEnvelope({ + messageId: id, version: 1, tsMs: now, + payload: new TextEncoder().encode(`invalid-inner-payload-${i}`), + }); + insert.run(process.env.PEER, PROTOCOL_MESSAGE, id, Buffer.from(payload), now, now, now - 1, 'probe'); + } +}); +seed(); +db.close(); +NODE + +remaining=250 +for _ in $(seq 1 75); do + remaining="$(DB="$DB" PREFIX="$PREFIX" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare('SELECT COUNT(*) AS n FROM protocol_outbox WHERE message_id LIKE ?').get(`${process.env.PREFIX}%`); +process.stdout.write(String(row.n)); db.close(); +NODE +)" + [[ "$remaining" -lt 250 ]] && break + sleep 1 +done +[[ "$remaining" -lt 250 ]] || fail "periodic scheduler did not start within 75s" +sleep 2 + +remaining="$(DB="$DB" PREFIX="$PREFIX" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare('SELECT COUNT(*) AS n FROM protocol_outbox WHERE message_id LIKE ?').get(`${process.env.PREFIX}%`); +process.stdout.write(String(row.n)); db.close(); +NODE +)" +processed=$((250 - remaining)) +[[ "$processed" -le 100 ]] || fail "one drain processed $processed rows (batch limit is 100)" +[[ "$processed" -gt 0 ]] || fail "no rows were processed" +echo "[#1580] PASS: first live drain was bounded ($processed/250 rows)" diff --git a/scripts/devnet-test-issue-1585-named-ka-swm.sh b/scripts/devnet-test-issue-1585-named-ka-swm.sh new file mode 100755 index 0000000000..d01f9b74e9 --- /dev/null +++ b/scripts/devnet-test-issue-1585-named-ka-swm.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Live order-stress regression for #1585. The first suite intentionally leaves +# named-KA SWM lifecycle residue; the subgraph RS suite then publishes on the +# same running devnet. The buggy family-wide named publish bundles/stomps that +# co-resident state and fails the second suite's merkle/cleanup assertions. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +[[ -d "${DEVNET_DIR:-$ROOT/.devnet}/node1" ]] || { + echo "[#1585] FAIL: start a six-node publisher-enabled devnet first" >&2 + exit 1 +} + +echo "[#1585] phase 1/2: create named-KA lifecycle residue" +pnpm test:devnet:ka-lifecycle-cli +echo "[#1585] phase 2/2: run subgraph publish/RS against the same residue" +pnpm test:devnet:pr1385-subgraph-rs +echo "[#1585] PASS: order-stressed named publish preserved co-resident SWM state"