From acfd572131a7662673440bbc1934b94ebad7a205 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:57:00 +0200 Subject: [PATCH 01/12] fix(core,mcp): escape all C0 control chars + DEL in RDF literals (#416) (#1617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(core,mcp): escape all C0 control chars + DEL in RDF literals (#416) escapeDkgRdfLiteral (core) and its byte-identical MCP copy escapeRdfLiteralBody only escaped the 7 ECHARs (\\ " \n \r \t \f \b), so NUL (0x00), VT (0x0B), DEL (0x7F) and the rest of 0x01-0x1F passed through raw — producing an invalid N-Triples/N-Quads STRING_LITERAL_QUOTED that the store rejected or stored corrupted. Both escapers now map every remaining U+0000-U+001F and U+007F to a \\uXXXX UCHAR (uppercase, 4-digit); the five with ECHAR short forms keep them so the byte output for already-covered chars is unchanged (stable merkle leaves). - packages/core/src/publisher-extension.ts — escapeDkgRdfLiteral - packages/mcp-dkg/src/tools/assertions.ts — escapeRdfLiteralBody (kept byte-identical) - packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts — golden fixture cases for NUL/VT/US/DEL + escaper assertion (per the drift-guard back-pointer) - packages/core/test/publisher-extension.test.ts — control-char coverage + ECHAR-stable assertion core suite 1186 passed; mcp conformance 30 passed. Fixes #416 Co-Authored-By: Claude Opus 4.8 * refactor(rdf): share literal escaping policy * fix: centralize RDF object normalization * build: follow workspace dependencies for runtime builds * test: cover Hermes RDF control escapes --------- Co-authored-by: Branimir Rakic Co-authored-by: Claude Opus 4.8 --- package.json | 2 +- ...st_rdf_object_normalization_conformance.py | 5 +++ packages/cli/package.json | 2 +- packages/core/package.json | 1 + packages/core/src/publisher-extension.ts | 34 ++++++---------- .../core/test/publisher-extension.test.ts | 5 +++ packages/core/tsconfig.json | 3 +- packages/mcp-dkg/package.json | 1 + packages/mcp-dkg/src/tools/assertions.ts | 36 +++++++---------- ...f-object-normalization-conformance.test.ts | 7 ++++ packages/mcp-dkg/tsconfig.json | 3 +- packages/rdf-utils/README.md | 4 ++ packages/rdf-utils/package.json | 38 ++++++++++++++++++ packages/rdf-utils/src/index.ts | 39 +++++++++++++++++++ packages/rdf-utils/test/rdf-literal.test.ts | 19 +++++++++ packages/rdf-utils/tsconfig.json | 9 +++++ packages/rdf-utils/vitest.config.ts | 7 ++++ pnpm-lock.yaml | 15 +++++++ 18 files changed, 180 insertions(+), 50 deletions(-) create mode 100644 packages/rdf-utils/README.md create mode 100644 packages/rdf-utils/package.json create mode 100644 packages/rdf-utils/src/index.ts create mode 100644 packages/rdf-utils/test/rdf-literal.test.ts create mode 100644 packages/rdf-utils/tsconfig.json create mode 100644 packages/rdf-utils/vitest.config.ts diff --git a/package.json b/package.json index 3ace5061c3..a98953f588 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "build": "node scripts/build.mjs", "build:packages": "turbo build", - "build:runtime:packages": "pnpm -r --filter @origintrail-official/dkg-core --filter @origintrail-official/dkg-storage --filter @origintrail-official/dkg-query --filter @origintrail-official/dkg-publisher --filter @origintrail-official/dkg-chain --filter @origintrail-official/dkg-epcis --filter @origintrail-official/dkg-okf --filter @origintrail-official/dkg-random-sampling --filter @origintrail-official/dkg-agent --filter @origintrail-official/dkg-graph-viz --filter @origintrail-official/dkg-node-ui --filter @origintrail-official/dkg-adapter-openclaw --filter @origintrail-official/dkg-adapter-hermes --filter @origintrail-official/kafka-plugin --filter @origintrail-official/dkg run build", + "build:runtime:packages": "pnpm -r --filter @origintrail-official/dkg-core... --filter @origintrail-official/dkg-storage... --filter @origintrail-official/dkg-query... --filter @origintrail-official/dkg-publisher... --filter @origintrail-official/dkg-chain... --filter @origintrail-official/dkg-epcis... --filter @origintrail-official/dkg-okf... --filter @origintrail-official/dkg-random-sampling... --filter @origintrail-official/dkg-agent... --filter @origintrail-official/dkg-graph-viz... --filter @origintrail-official/dkg-node-ui... --filter @origintrail-official/dkg-adapter-openclaw... --filter @origintrail-official/dkg-adapter-hermes... --filter @origintrail-official/kafka-plugin... --filter @origintrail-official/dkg... run build", "build:runtime": "pnpm run build:runtime:packages && pnpm --filter @origintrail-official/dkg-node-ui run build:ui", "test": "turbo test && pnpm run test:scripts", "test:scripts": "node --test scripts/lib/__tests__/*.test.mjs", diff --git a/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py b/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py index 703c276bbd..25b972aaaa 100644 --- a/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py +++ b/packages/adapter-hermes/pytests/test_rdf_object_normalization_conformance.py @@ -47,6 +47,11 @@ ("a\rb", '"a\\rb"'), ("a\fb", '"a\\fb"'), ("a\bb", '"a\\bb"'), + # remaining ASCII controls -> UCHAR escapes (#416) + ("nul\u0000x", '"nul\\u0000x"'), + ("vt\u000bx", '"vt\\u000Bx"'), + ("us\u001fx", '"us\\u001Fx"'), + ("del\u007fx", '"del\\u007Fx"'), ] diff --git a/packages/cli/package.json b/packages/cli/package.json index 44d892b593..8cf51512cb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -15,7 +15,7 @@ } }, "scripts": { - "prebuild": "pnpm --filter @origintrail-official/dkg-adapter-openclaw run build && pnpm --filter @origintrail-official/dkg-adapter-hermes run build && pnpm --filter @origintrail-official/dkg-mcp run build && pnpm --filter @origintrail-official/dkg-okf run build", + "prebuild": "pnpm -r --filter @origintrail-official/dkg-adapter-openclaw... --filter @origintrail-official/dkg-adapter-hermes... --filter @origintrail-official/dkg-mcp... --filter @origintrail-official/dkg-okf... run build", "build": "tsc && node ../../scripts/copy-cli-runtime-assets.mjs", "prepack": "node ../../scripts/copy-cli-runtime-assets.mjs", "benchmark:catchup-runner": "node scripts/catchup-runner-benchmark.cjs", diff --git a/packages/core/package.json b/packages/core/package.json index fd155c745c..526b00bbb3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -25,6 +25,7 @@ "clean": "rm -rf dist tsconfig.tsbuildinfo" }, "dependencies": { + "@origintrail-official/dkg-rdf-utils": "workspace:*", "@libp2p/autonat": "^3.0.20", "@libp2p/bootstrap": "^12.0.22", "@libp2p/circuit-relay-v2": "^4.2.5", diff --git a/packages/core/src/publisher-extension.ts b/packages/core/src/publisher-extension.ts index 69101216c9..a87560903d 100644 --- a/packages/core/src/publisher-extension.ts +++ b/packages/core/src/publisher-extension.ts @@ -1,3 +1,9 @@ +import { + escapeRdfLiteral, + isRdfTerm, + normalizeRdfObject, +} from '@origintrail-official/dkg-rdf-utils'; + export interface DkgPublisherExtensionQuadInput { subject: unknown; predicate: unknown; @@ -146,25 +152,14 @@ export function normalizeDkgPublisherQuads( })); } -// NOTE: the MCP adapter inlines a byte-for-byte copy of this normalizer (it is -// deliberately dep-light and does not import dkg-core) as `normalizeRdfObject` in -// `packages/mcp-dkg/src/tools/assertions.ts`. If you change the behavior here, -// update that copy AND its golden fixture in -// `packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts` so the -// public `dkg_knowledge_asset_create({quads})` object contract stays identical -// across the MCP / OpenClaw / Hermes adapters. +// Core and MCP preserve their public names as thin compatibility wrappers around +// the complete dependency-free normalizer in @origintrail-official/dkg-rdf-utils. export function normalizeDkgPublisherObject(value: unknown): string { - const raw = String(value ?? ''); - if (isDkgRdfTerm(raw)) return raw; - return `"${escapeDkgRdfLiteral(raw)}"`; + return normalizeRdfObject(value); } export function isDkgRdfTerm(value: string): boolean { - return ( - /^(?:https?:\/\/|urn:|did:)/i.test(value) || - value.startsWith('_:') || - value.startsWith('"') - ); + return isRdfTerm(value); } /** @@ -172,14 +167,7 @@ export function isDkgRdfTerm(value: string): boolean { * Returns only the escaped body; callers wrap it in quotes. */ export function escapeDkgRdfLiteral(value: string): string { - return value - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/\t/g, '\\t') - .replace(/\f/g, '\\f') - .replace(/\x08/g, '\\b'); + return escapeRdfLiteral(value); } export { diff --git a/packages/core/test/publisher-extension.test.ts b/packages/core/test/publisher-extension.test.ts index a55f4d3ace..043c76b665 100644 --- a/packages/core/test/publisher-extension.test.ts +++ b/packages/core/test/publisher-extension.test.ts @@ -46,6 +46,11 @@ describe('DkgPublisherExtension', () => { '"42"^^', ); expect(escapeDkgRdfLiteral('a "quote"\nnext')).toBe('a \\"quote\\"\\nnext'); + // #416: non-ECHAR C0 controls (NUL, VT, unit-sep) + DEL must be escaped as + // \uXXXX, not passed through raw (raw controls = invalid N-Triples literal). + expect(escapeDkgRdfLiteral('n\u0000v\u000Bu\u001Fd\u007F')).toBe('n\\u0000v\\u000Bu\\u001Fd\\u007F'); + // ECHAR short forms are preserved (stable merkle output). + expect(escapeDkgRdfLiteral('t\tn\nr\rf\fb\b')).toBe('t\\tn\\nr\\rf\\fb\\b'); }); it('normalizes full quads without changing URI, literal, or blank-node RDF terms', () => { diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index d231bbc57e..d99ceee5ab 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -5,5 +5,6 @@ "rootDir": "src", "composite": true }, - "include": ["src"] + "include": ["src"], + "references": [{ "path": "../rdf-utils" }] } diff --git a/packages/mcp-dkg/package.json b/packages/mcp-dkg/package.json index 5006833365..7b373ca3c1 100644 --- a/packages/mcp-dkg/package.json +++ b/packages/mcp-dkg/package.json @@ -42,6 +42,7 @@ "clean": "rm -rf dist" }, "dependencies": { + "@origintrail-official/dkg-rdf-utils": "workspace:*", "@modelcontextprotocol/sdk": "^1", "zod": "^3.25", "yaml": "^2.6.0" diff --git a/packages/mcp-dkg/src/tools/assertions.ts b/packages/mcp-dkg/src/tools/assertions.ts index 615f289d0d..a0894ec28c 100644 --- a/packages/mcp-dkg/src/tools/assertions.ts +++ b/packages/mcp-dkg/src/tools/assertions.ts @@ -15,6 +15,11 @@ * assertion name. */ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { + escapeRdfLiteral, + isRdfTerm as isSharedRdfTerm, + normalizeRdfObject as normalizeSharedRdfObject, +} from '@origintrail-official/dkg-rdf-utils'; import { z } from 'zod'; import type { DkgClient } from '../client.js'; import { DkgHttpError } from '../client.js'; @@ -105,12 +110,10 @@ function validateAssertionName(name: string): { valid: boolean; reason?: string /** * Object-term normalizer for the `dkg_knowledge_asset_create` one-shot `quads` - * path. Replicated verbatim from `@origintrail-official/dkg-core` - * `normalizeDkgPublisherObject` / `isDkgRdfTerm` / `escapeDkgRdfLiteral` - * (`packages/core/src/publisher-extension.ts:212-239`) — MCP does not depend on - * dkg-core (it is deliberately dependency-light; it inlines `validateAssertionName` - * and the share-warning constants for the same reason), and core does not export the - * normalizer from its package entry, so the rule is inlined here. OpenClaw + Hermes + * path. The complete dependency-free normalization boundary (term classification, + * literal escaping, and quote wrapping) lives in + * `@origintrail-official/dkg-rdf-utils`. MCP remains independent of the full + * dkg-core runtime. OpenClaw + Hermes * route the SAME create-tool `quads` shape through that core normalizer, so a bare * literal object must auto-quote identically across all three runtimes * (portable-agent parity). A value that is already an http(s)/urn/did URI, a blank @@ -120,33 +123,20 @@ function validateAssertionName(name: string): { valid: boolean; reason?: string * * DRIFT GUARD: the public contract of these three functions is pinned by a GOLDEN * conformance fixture in `packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts` - * whose expected strings are HAND-WRITTEN (not derived from this impl). If core's - * `normalizeDkgPublisherObject`/`escapeDkgRdfLiteral` changes, update this inline - * copy AND that fixture together (core carries a back-pointer to that test). + * whose expected strings are HAND-WRITTEN (not derived from this impl). * * Exported so the conformance test can pin them directly. */ export function isRdfTerm(value: string): boolean { - return ( - /^(?:https?:\/\/|urn:|did:)/i.test(value) || - value.startsWith('_:') || - value.startsWith('"') - ); + return isSharedRdfTerm(value); } export function escapeRdfLiteralBody(value: string): string { - return value - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/\t/g, '\\t') - .replace(/\f/g, '\\f') - .replace(/\x08/g, '\\b'); + return escapeRdfLiteral(value); } export function normalizeRdfObject(value: string): string { - return isRdfTerm(value) ? value : `"${escapeRdfLiteralBody(value)}"`; + return normalizeSharedRdfObject(value); } function resolveProject( diff --git a/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts b/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts index 538613346c..5705f71e1e 100644 --- a/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts +++ b/packages/mcp-dkg/test/rdf-object-normalization-conformance.test.ts @@ -65,6 +65,11 @@ const GOLDEN: Array<{ input: string; expected: string; note: string }> = [ { input: 'form\ffeed', expected: '"form\\ffeed"', note: 'form-feed escaped' }, { input: 'bs\bhere', expected: '"bs\\bhere"', note: 'backspace escaped' }, { input: 'a"b\nc', expected: '"a\\"b\\nc"', note: 'quote + newline together' }, + // ── non-ECHAR control chars (#416): must become \uXXXX, not pass through raw ── + { input: 'nul\u0000x', expected: '"nul\\u0000x"', note: 'NUL → \\u0000 UCHAR' }, + { input: 'vt\u000Bx', expected: '"vt\\u000Bx"', note: 'vertical tab (0x0B) → \\u000B' }, + { input: 'us\u001Fx', expected: '"us\\u001Fx"', note: 'unit separator (0x1F) → \\u001F' }, + { input: 'del\u007Fx', expected: '"del\\u007Fx"', note: 'DEL (0x7F) → \\u007F' }, ]; describe('rdf-object-normalization conformance (dkg_knowledge_asset_create quads — cross-adapter parity)', () => { @@ -90,5 +95,7 @@ describe('rdf-object-normalization conformance (dkg_knowledge_asset_create quads it('escapeRdfLiteralBody escapes the N-Triples ECHAR set (body only, no surrounding quotes)', () => { expect(escapeRdfLiteralBody('a\\b"c\nd\re\tf\fg\bh')).toBe('a\\\\b\\"c\\nd\\re\\tf\\fg\\bh'); expect(escapeRdfLiteralBody('plain')).toBe('plain'); + // #416: non-ECHAR C0 controls + DEL become \uXXXX (uppercase, 4-digit). + expect(escapeRdfLiteralBody('a\u0000b\u000Bc\u007Fd')).toBe('a\\u0000b\\u000Bc\\u007Fd'); }); }); diff --git a/packages/mcp-dkg/tsconfig.json b/packages/mcp-dkg/tsconfig.json index d231bbc57e..d99ceee5ab 100644 --- a/packages/mcp-dkg/tsconfig.json +++ b/packages/mcp-dkg/tsconfig.json @@ -5,5 +5,6 @@ "rootDir": "src", "composite": true }, - "include": ["src"] + "include": ["src"], + "references": [{ "path": "../rdf-utils" }] } diff --git a/packages/rdf-utils/README.md b/packages/rdf-utils/README.md new file mode 100644 index 0000000000..b5f5fd802d --- /dev/null +++ b/packages/rdf-utils/README.md @@ -0,0 +1,4 @@ +# DKG RDF utilities + +Dependency-free RDF serialization helpers shared by DKG packages that must not +depend on the full DKG core runtime. diff --git a/packages/rdf-utils/package.json b/packages/rdf-utils/package.json new file mode 100644 index 0000000000..3fb8e4cdb1 --- /dev/null +++ b/packages/rdf-utils/package.json @@ -0,0 +1,38 @@ +{ + "name": "@origintrail-official/dkg-rdf-utils", + "version": "10.0.6", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "scripts": { + "build": "tsc", + "test": "vitest run", + "test:coverage": "vitest run --coverage", + "clean": "rm -rf dist tsconfig.tsbuildinfo" + }, + "devDependencies": { + "@vitest/coverage-v8": "^4.0.18", + "vitest": "^4.0.18" + }, + "publishConfig": { + "access": "public" + }, + "files": [ + "dist", + "README.md" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/OriginTrail/dkg.git", + "directory": "packages/rdf-utils" + } +} diff --git a/packages/rdf-utils/src/index.ts b/packages/rdf-utils/src/index.ts new file mode 100644 index 0000000000..9b0a8cc0a0 --- /dev/null +++ b/packages/rdf-utils/src/index.ts @@ -0,0 +1,39 @@ +/** N-Triples ECHAR short forms, keyed by the raw character. */ +const RDF_LITERAL_SHORT_ESCAPES: Readonly> = Object.freeze({ + '\b': '\\b', + '\t': '\\t', + '\n': '\\n', + '\f': '\\f', + '\r': '\\r', + '"': '\\"', + '\\': '\\\\', +}); + +const RDF_LITERAL_ESCAPE_PATTERN = /["\\\u0000-\u001F\u007F]/g; + +/** + * Escape a plain-text string for use as an RDF/N-Triples literal body. + * Returns only the escaped body; callers add the surrounding quotes. + */ +export function escapeRdfLiteral(value: string): string { + return value.replace(RDF_LITERAL_ESCAPE_PATTERN, (character) => { + const shortEscape = RDF_LITERAL_SHORT_ESCAPES[character]; + if (shortEscape !== undefined) return shortEscape; + return `\\u${character.charCodeAt(0).toString(16).toUpperCase().padStart(4, '0')}`; + }); +} + +/** Return whether a string already represents an RDF term accepted by DKG publishers. */ +export function isRdfTerm(value: string): boolean { + return ( + /^(?:https?:\/\/|urn:|did:)/i.test(value) || + value.startsWith('_:') || + value.startsWith('"') + ); +} + +/** Preserve RDF terms and quote/escape every other value as a plain literal. */ +export function normalizeRdfObject(value: unknown): string { + const raw = String(value ?? ''); + return isRdfTerm(raw) ? raw : `"${escapeRdfLiteral(raw)}"`; +} diff --git a/packages/rdf-utils/test/rdf-literal.test.ts b/packages/rdf-utils/test/rdf-literal.test.ts new file mode 100644 index 0000000000..bbc5ee637a --- /dev/null +++ b/packages/rdf-utils/test/rdf-literal.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { escapeRdfLiteral, isRdfTerm, normalizeRdfObject } from '../src/index.js'; + +describe('escapeRdfLiteral', () => { + it('escapes quotes, backslashes, ECHAR controls, remaining C0 controls, and DEL', () => { + expect(escapeRdfLiteral('q"\\b\bt\tn\nf\fr\rnul\u0000vt\u000Bus\u001Fdel\u007F')).toBe( + 'q\\"\\\\b\\bt\\tn\\nf\\fr\\rnul\\u0000vt\\u000Bus\\u001Fdel\\u007F', + ); + }); + + it('owns the complete dependency-free RDF object normalization boundary', () => { + expect(isRdfTerm('urn:test:entity')).toBe(true); + expect(isRdfTerm('_:blank')).toBe(true); + expect(isRdfTerm('plain')).toBe(false); + expect(normalizeRdfObject('urn:test:entity')).toBe('urn:test:entity'); + expect(normalizeRdfObject('a "quote"\u0000')).toBe('"a \\"quote\\"\\u0000"'); + expect(normalizeRdfObject(null)).toBe('""'); + }); +}); diff --git a/packages/rdf-utils/tsconfig.json b/packages/rdf-utils/tsconfig.json new file mode 100644 index 0000000000..d231bbc57e --- /dev/null +++ b/packages/rdf-utils/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "composite": true + }, + "include": ["src"] +} diff --git a/packages/rdf-utils/vitest.config.ts b/packages/rdf-utils/vitest.config.ts new file mode 100644 index 0000000000..43e56f45f3 --- /dev/null +++ b/packages/rdf-utils/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d31b0a2e05..d5e09d3592 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -654,6 +654,9 @@ importers: '@opentelemetry/api': specifier: ^1.9.1 version: 1.9.1 + '@origintrail-official/dkg-rdf-utils': + specifier: workspace:* + version: link:../rdf-utils js-yaml: specifier: ^4.1.1 version: 4.1.1 @@ -846,6 +849,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1 version: 1.27.1(zod@3.25.76) + '@origintrail-official/dkg-rdf-utils': + specifier: workspace:* + version: link:../rdf-utils yaml: specifier: ^2.6.0 version: 2.8.3 @@ -1109,6 +1115,15 @@ importers: specifier: ^4.0.18 version: 4.0.18(@opentelemetry/api@1.9.1)(@types/node@22.19.11)(happy-dom@20.8.9(bufferutil@4.1.0)(utf-8-validate@5.0.10))(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) + packages/rdf-utils: + devDependencies: + '@vitest/coverage-v8': + specifier: ^4.0.18 + version: 4.0.18(vitest@4.0.18(@opentelemetry/api@1.9.1)(@types/node@22.19.11)(happy-dom@20.8.9(bufferutil@4.1.0)(utf-8-validate@5.0.10))(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) + vitest: + specifier: ^4.0.18 + version: 4.0.18(@opentelemetry/api@1.9.1)(@types/node@22.19.11)(happy-dom@20.8.9(bufferutil@4.1.0)(utf-8-validate@5.0.10))(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) + packages/storage: dependencies: '@origintrail-official/dkg-core': From d293bba03aae0676cdd3de2a79da81ee7002389b Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:57:17 +0200 Subject: [PATCH 02/12] fix(publisher): stop the doomed post-confirm chain.verify on V10 sync publish (#1575) (#1618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(publisher): stop the doomed post-confirm chain.verify on V10 sync publish (#1575) Every confirmed V10 publish with a target context graph ran a legacy V9-era explicit chain.verify() fallback in the publish tail. On V10 the KC is ALREADY registered to the context graph inside publishDirect via a Hub-authorized internal call to ContextGraphs.registerKnowledgeAsset (EOAs can't call it), so the explicit verify always reverted with "Only Contracts in Hub" / CALL_EXCEPTION — a guaranteed-to-fail on-chain call plus an estimateGas round-trip and per-wallet serializer occupancy on EVERY publish. Skip it: registration is already done. `registered` stays true so the GH#842 last-writer-wins per-cgId data promotion still runs (removing that would strand published KCs from RS random sampling). Only the verify attempt is removed. - packages/publisher/src/dkg-publisher.ts — replace the verify/sign/estimateGas block with an unconditional registered=true + a debug log. - packages/publisher/test/dkg-publisher.test.ts — assert chain.verify is NOT called on a confirmed V10 publish (real EVM adapter, hardhat). publisher suite 1482 passed; the new assertion is green against the real chain. Fixes #1575 Co-Authored-By: Claude Opus 4.8 * test(publisher): cover SWM verify removal --------- Co-authored-by: Branimir Rakic Co-authored-by: Claude Opus 4.8 --- packages/publisher/src/dkg-publisher.ts | 62 +++---------------- packages/publisher/test/dkg-publisher.test.ts | 26 +++++++- 2 files changed, 35 insertions(+), 53 deletions(-) diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index d379456193..1f45c5b9ae 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -1712,58 +1712,17 @@ export class DKGPublisher implements Publisher { // `ctxGraphId ?? chainCgId`. const targetCgId = ctxGraphId ?? chainCgId; if (targetCgId && publishResult.status === 'confirmed' && publishResult.onChainResult) { - // V10 publishDirect already registers the KC to the context graph - // via an internal call to ContextGraphs.registerKnowledgeAsset - // (Hub-authorized only — EOAs cannot call it directly). The legacy - // V9 flow required a separate addBatchToContextGraph tx; that path - // is no longer available. Attempt the explicit verify call as a - // fallback for non-V10 chains, but treat "Only Contracts in Hub" - // rejections as success (V10 already handled it). - let registered = false; - if (typeof this.chain.verify === 'function') { - let participantSigs = options?.contextGraphSignatures ?? []; - if (participantSigs.length === 0 && typeof this.chain.signMessage === 'function') { - const identityId = this.publisherNodeIdentityId; - if (identityId > 0n) { - const digest = ethers.solidityPackedKeccak256( - ['uint256', 'bytes32'], - [BigInt(targetCgId), ethers.hexlify(publishResult.merkleRoot)], - ); - const sig = await this.chain.signMessage(ethers.getBytes(digest)); - participantSigs = [{ identityId, ...sig }]; - } - } - - const sortedSigs = [...participantSigs] - .sort((a, b) => (a.identityId < b.identityId ? -1 : a.identityId > b.identityId ? 1 : 0)) - .filter((s, i, arr) => i === 0 || s.identityId !== arr[i - 1].identityId); + // V10 publishDirect already registered the KC to the context graph + // inside publishDirect, via a Hub-authorized internal call to + // ContextGraphs.registerKnowledgeAsset (EOAs cannot call it directly), + // which emits KnowledgeAssetRegisteredToContextGraph. The legacy V9 + // explicit chain.verify() fallback that used to run here always reverted + // on V10 ("Only Contracts in Hub" / CALL_EXCEPTION) — i.e. a doomed + // on-chain call plus an estimateGas round-trip, and serializer occupancy, + // on EVERY confirmed publish (#1575). Registration is already done, so + // skip the verify attempt entirely and proceed to the data promotion. + this.log.debug(ctx, `V10 auto-registered KC to context graph ${targetCgId}; explicit verify skipped`); - try { - const txResult = await this.chain.verify({ - contextGraphId: BigInt(targetCgId), - batchId: publishResult.onChainResult.batchId, - merkleRoot: publishResult.merkleRoot, - signerSignatures: sortedSigs, - }); - if (txResult && typeof txResult === 'object' && 'success' in txResult && txResult.success) { - registered = true; - this.log.info(ctx, `Batch ${publishResult.onChainResult.batchId} verified on context graph ${targetCgId}`); - } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - // V10 publishDirect handles registration internally via a - // Hub-authorized call. Any revert here (typically - // "Only Contracts in Hub" / CALL_EXCEPTION) means the - // explicit verify path is not applicable — treat as success. - registered = true; - this.log.info(ctx, `Explicit verify not needed (V10 auto-registered): ${msg.slice(0, 120)}`); - } - } else { - registered = true; - this.log.info(ctx, `No verify function on chain adapter — assuming V10 auto-registration for context graph ${targetCgId}`); - } - - if (registered) { const ctxDataGraph = contextGraphDataUri(contextGraphId, targetCgId); const ctxMetaGraph = contextGraphMetaUri(contextGraphId, targetCgId); const defaultDataGraph = this.graphManager.dataGraphUri(contextGraphId); @@ -1883,7 +1842,6 @@ export class DKGPublisher implements Publisher { this.log.info(ctx, `Promoted ${publishResult.kaManifest.length} KAs from default graph to context graph ${targetCgId}`); } }); - } } // SWM cleanup: ALWAYS remove published triples from SWM after chain confirmation. diff --git a/packages/publisher/test/dkg-publisher.test.ts b/packages/publisher/test/dkg-publisher.test.ts index 2a7fc9cc7f..667f07e94a 100644 --- a/packages/publisher/test/dkg-publisher.test.ts +++ b/packages/publisher/test/dkg-publisher.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, beforeAll, afterAll, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, beforeAll, afterAll, afterEach, vi } from 'vitest'; import { OxigraphStore } from '@origintrail-official/dkg-storage'; import { EVMChainAdapter } from '@origintrail-official/dkg-chain'; import { @@ -336,6 +336,30 @@ describe('DKGPublisher', () => { expect(result.onChainResult!.endKAId).toBeDefined(); }); + it('does NOT call chain.verify after a confirmed V10 SWM publish (#1575)', async () => { + const quad = q(ENTITY, 'http://schema.org/name', '"ImageBot"'); + await publisher.share(CONTEXT_GRAPH, [quad], { + publisherPeerId: 'peer-no-legacy-verify', + localOnly: true, + }); + await seedContextGraphRegistration(store, CONTEXT_GRAPH); + + const verifySpy = vi.spyOn(chain, 'verify'); + const result = await publisher.publishFromSharedMemory(CONTEXT_GRAPH, 'all', { + onChainContextGraphId: CONTEXT_GRAPH, + precomputedAttestation: await buildSeal({ + quads: [{ ...quad, graph: GRAPH }], + author: _author, + contextGraphId: CONTEXT_GRAPH, + ctx: { provider: _provider, kav10Address: _kav10Address }, + }), + }); + + expect(result.status).toBe('confirmed'); + expect(result.onChainResult).toBeDefined(); + expect(verifySpy).not.toHaveBeenCalled(); + }); + it('generates address-based UAL format', async () => { const result = await publishWS({ contextGraphId: CONTEXT_GRAPH, From 61cdf9a9a52fa7fdf7c7b9924342abad39be7d5d Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:57:36 +0200 Subject: [PATCH 03/12] =?UTF-8?q?fix(query):=20DISTINCT=20on=20read-both?= =?UTF-8?q?=20graph=20unions=20=E2=80=94=20collapse=20dual-homed=20triples?= =?UTF-8?q?=20(#1161,=20#1270)=20(#1619)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(query): DISTINCT on read-both graph unions — collapse dual-homed triples (#1270) A scoped query (no explicit view) reads a UNION across the canonical root graph and the per-cgId / per-KA …/_verifiable_memory partitions (wrapWithGraphUnion). Finalization intentionally DUAL-HOMES a confirmed triple — PR #1098 made the per-KA VM partitions queryable for pre-subscribed peer recovery, and promote mirrors the canonical quads into the root graph — so the SAME triple matches in two union branches and returns duplicate solution rows. This surfaced as the flaky e2e-finalization failure "A enshrines; B promotes": after B promotes, SELECT ?name WHERE { schema:name ?name } returned 2 bindings instead of 1. (Reproduced at the query layer: ROOT+VM → 2 bindings.) Fix: emit SELECT DISTINCT for the multi-graph union branch in wrapWithGraphUnion, collapsing mirror duplicates. No-op for a single graph (a graph is a set), so only the union path needs it; CONSTRUCT/ASK/DESCRIBE are untouched. The data is correctly dual-homed — this is a read-path dedup, not a change to the writes. Hardened the existing dedup probe into a strict regression test (each dual-home layout must yield exactly ONE binding). Full query suite green (285). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 2f0195cb2e52d7ababcdae4cad2076fbb71fd936) * fix(query): dedupe mirrors without changing bag semantics * test(agent): expect deduplicated VM mirrors * fix: constrain VM mirror dedup rewrites --------- Co-authored-by: Branimir Rakic Co-authored-by: Claude Opus 4.8 (1M context) --- .../test/finalization-promote-extra.test.ts | 15 +- packages/query/src/dkg-query-engine.ts | 189 +++++++++++++----- packages/query/test/read-both-dedup.test.ts | 99 +++++++++ 3 files changed, 249 insertions(+), 54 deletions(-) create mode 100644 packages/query/test/read-both-dedup.test.ts diff --git a/packages/agent/test/finalization-promote-extra.test.ts b/packages/agent/test/finalization-promote-extra.test.ts index 01d90b8184..acfcb1c20a 100644 --- a/packages/agent/test/finalization-promote-extra.test.ts +++ b/packages/agent/test/finalization-promote-extra.test.ts @@ -439,16 +439,15 @@ describe('A-4: e2e — agent.publish() data lands in canonical (data) view post- // GH #1264 promotion: a confirmed one-shot `publish()` now stores its public // data in BOTH the per-KA verifiable-memory graph (the publish write) AND the // scoped RS-prover graph `/context/` (promoteConfirmedKCToScopedGraph). - // The verifiable-memory view unions both (dkg-query-engine.ts re-includes the - // per-cgId data graphs for #1098), so the SAME triple is observed once per - // source graph — two rows. The invariant is unchanged: the confirmed publish - // is immediately observable via VM with the correct value. Assert that the two - // rows are exactly the two known sources AND carry only the published value, - // so a third copy (count) or a wrong-member/garbage row (distinct set) still fails. + // The verifiable-memory read-both includes both (dkg-query-engine.ts + // re-includes the per-cgId data graphs for #1098), but collapses an identical + // full solution mapping already produced by an earlier mirror graph. The + // canonical triple is therefore observed once without applying DISTINCT to + // the caller projection (which would also erase legitimate bag multiplicity). expect( vmQr.bindings.length, - 'VM view observes the confirmed publish in its two source graphs (per-KA VM graph + scoped #1264 promotion)', - ).toBe(2); + 'VM view deduplicates the identical per-KA VM + scoped #1264 mirror', + ).toBe(1); expect( new Set(vmQr.bindings.map((b) => b['o'])), 'every VM-view row must carry the published value (no wrong-member or stale row)', diff --git a/packages/query/src/dkg-query-engine.ts b/packages/query/src/dkg-query-engine.ts index aafc504419..63306a2dc2 100644 --- a/packages/query/src/dkg-query-engine.ts +++ b/packages/query/src/dkg-query-engine.ts @@ -377,7 +377,8 @@ export class DKGQueryEngine implements QueryEngine { // Per-KA VM: read-both the published per-KA …/_verifiable_memory/{addr}/{number} + root. const vmGraphsInc = await this.discoverGraphsByPrefix(`${dataGraph}/_verifiable_memory/`); const dataSparql = vmGraphsInc.length > 0 - ? (wrapWithGraphUnion(sparql, [dataGraph, ...vmGraphsInc]) ?? wrapWithGraph(sparql, dataGraph)) + ? (this.wrapVerifiableMemoryGraphSet(sparql, [dataGraph, ...vmGraphsInc]) + ?? wrapWithGraph(sparql, dataGraph)) : wrapWithGraph(sparql, dataGraph); // Per-KA SWM: union the discovered …/_shared_memory/{addr}/{number} graphs. const swmGraphs = await this.discoverGraphsByPrefix(`${sharedMemoryGraph}/`); @@ -399,7 +400,8 @@ export class DKGQueryEngine implements QueryEngine { // Per-KA VM: read-both the published per-KA …/_verifiable_memory/{addr}/{number} + root. const vmGraphs = await this.discoverGraphsByPrefix(`${dataGraph}/_verifiable_memory/`); effectiveSparql = vmGraphs.length > 0 - ? (wrapWithGraphUnion(sparql, [dataGraph, ...vmGraphs]) ?? wrapWithGraph(sparql, dataGraph)) + ? (this.wrapVerifiableMemoryGraphSet(sparql, [dataGraph, ...vmGraphs]) + ?? wrapWithGraph(sparql, dataGraph)) : wrapWithGraph(sparql, dataGraph); } } @@ -587,9 +589,23 @@ export class DKGQueryEngine implements QueryEngine { return this.execAndNormalize(wrapWithGraph(effectiveSparql, allGraphs[0])); } + if (view === 'verifiable-memory') { + const rewritten = this.wrapVerifiableMemoryGraphSet(effectiveSparql, allGraphs); + if (rewritten !== null) return this.execAndNormalize(rewritten); + } + return this.queryMultipleGraphs(effectiveSparql, allGraphs); } + /** Canonical graph rewrite for root + per-KA/per-cgId verifiable-memory reads. */ + private wrapVerifiableMemoryGraphSet(sparql: string, graphs: string[]): string | null { + if (graphs.length === 0) return sparql; + if (graphs.length === 1) return wrapWithGraph(sparql, graphs[0]); + return wrapWithDeduplicatedGraphValues(sparql, graphs) + ?? wrapWithGraphValues(sparql, graphs) + ?? wrapWithGraphUnion(sparql, graphs); + } + private async queryMultipleGraphs(sparql: string, graphs: string[]): Promise { if (graphs.length === 0) return { bindings: [] }; if (graphs.length === 1) { @@ -2450,6 +2466,86 @@ function wrapWithGraphUnion(sparql: string, graphUris: string[]): string | null * view-specific name so a user query is astronomically unlikely to bind it; * a collision is nonetheless detected and declined (never silently clamped). */ const VIEW_GRAPH_SENTINEL = '?__dkgViewGraph'; +const DEDUP_GRAPH_SENTINEL = '?__dkgDedupGraph'; +const DEDUP_RANK_SENTINEL = '?__dkgDedupRank'; +const DEDUP_PRIOR_GRAPH_SENTINEL = '?__dkgDedupPriorGraph'; +const DEDUP_PRIOR_RANK_SENTINEL = '?__dkgDedupPriorRank'; + +function wrapWithProjectedGraphSubselect( + sparql: string, + graphUris: string[], + helperVariables: string[], + buildGraphPattern: (inner: string, graphs: string[]) => string, + acceptsInner: (inner: string) => boolean = () => true, +): string | null { + if (hasGraphClause(sparql)) return sparql; + if (graphUris.length === 0) return sparql; + + const braceStart = findWhereBraceStart(sparql); + if (braceStart === -1) return null; + const braceEnd = findMatchingCloseBrace(sparql, braceStart); + if (braceEnd === -1) return null; + + const before = sparql.slice(0, braceStart + 1); + const inner = sparql.slice(braceStart + 1, braceEnd); + const after = sparql.slice(braceEnd); + const graphs = [...new Set(graphUris)]; + + if (graphs.length === 1) { + return `${before} GRAPH <${assertSafeIri(graphs[0])}> { ${inner} } ${after}`; + } + if (/\bUNION\b/i.test(inner) || !acceptsInner(inner)) return null; + + const helperNames = new Set(helperVariables.map((variable) => variable.slice(1))); + if (collectQueryVariables(sparql).some((variable) => helperNames.has(variable.slice(1)))) { + return null; + } + + const innerVars = collectQueryVariables(inner); + if (innerVars.length === 0) return null; + + const graphPattern = buildGraphPattern(inner, graphs); + return `${before} { SELECT ${innerVars.join(' ')} WHERE { ${graphPattern} } } ${after}`; +} + +/** + * Run one graph pattern across an ordered graph set while suppressing only a + * solution mapping already produced by an earlier graph. The comparison uses + * every variable bound by the caller's inner pattern, before the caller's + * projection runs. Thus an identical mirrored triple is emitted once, while + * distinct triples that both project to the same `?s` still produce two rows. + * + * Helper graph/rank variables are hidden inside a sub-SELECT, preserving + * `SELECT *` and caller DISTINCT semantics. Unsupported/colliding query shapes + * return null so the existing generic multi-graph fallback remains available. + */ +function wrapWithDeduplicatedGraphValues(sparql: string, graphUris: string[]): string | null { + return wrapWithProjectedGraphSubselect( + sparql, + graphUris, + [ + DEDUP_GRAPH_SENTINEL, + DEDUP_RANK_SENTINEL, + DEDUP_PRIOR_GRAPH_SENTINEL, + DEDUP_PRIOR_RANK_SENTINEL, + ], + (inner, graphs) => { + const rows = graphs + .map((graph, rank) => `(<${assertSafeIri(graph)}> ${rank})`) + .join(' '); + return [ + `VALUES (${DEDUP_GRAPH_SENTINEL} ${DEDUP_RANK_SENTINEL}) { ${rows} }`, + `GRAPH ${DEDUP_GRAPH_SENTINEL} { ${inner} }`, + 'FILTER NOT EXISTS {', + ` VALUES (${DEDUP_PRIOR_GRAPH_SENTINEL} ${DEDUP_PRIOR_RANK_SENTINEL}) { ${rows} }`, + ` FILTER (${DEDUP_PRIOR_RANK_SENTINEL} < ${DEDUP_RANK_SENTINEL})`, + ` GRAPH ${DEDUP_PRIOR_GRAPH_SENTINEL} { ${inner} }`, + '}', + ].join(' '); + }, + isDedupSafeBasicGraphPattern, + ); +} /** * Wrap a query so it runs over a set of named graphs in ONE execution using a @@ -2480,52 +2576,53 @@ const VIEW_GRAPH_SENTINEL = '?__dkgViewGraph'; * nowhere to hide the sentinel from a `SELECT *` projection). */ function wrapWithGraphValues(sparql: string, graphUris: string[]): string | null { - if (hasGraphClause(sparql)) return sparql; - if (graphUris.length === 0) return sparql; - - const braceStart = findWhereBraceStart(sparql); - if (braceStart === -1) return null; - const braceEnd = findMatchingCloseBrace(sparql, braceStart); - if (braceEnd === -1) return null; - - const before = sparql.slice(0, braceStart + 1); - const inner = sparql.slice(braceStart + 1, braceEnd); - const after = sparql.slice(braceEnd); - - if (graphUris.length === 1) { - return `${before} GRAPH <${assertSafeIri(graphUris[0])}> { ${inner} } ${after}`; - } - - // An inner top-level UNION stays on the per-graph fallback (#789): merging - // that shape across graphs is form-aware there, and this keeps that path and - // its tests unchanged. Same guard `wrapWithGraphUnion` uses. - if (/\bUNION\b/i.test(inner)) return null; - - // Never clamp a variable the user actually uses — fall back instead. - const sentinelName = VIEW_GRAPH_SENTINEL.slice(1); - if (collectQueryVariables(sparql).some((v) => v.slice(1) === sentinelName)) { - return null; - } + return wrapWithProjectedGraphSubselect( + sparql, + graphUris, + [VIEW_GRAPH_SENTINEL], + (inner, graphs) => { + const values = graphs.map((graph) => `<${assertSafeIri(graph)}>`).join(' '); + return `VALUES ${VIEW_GRAPH_SENTINEL} { ${values} } GRAPH ${VIEW_GRAPH_SENTINEL} { ${inner} }`; + }, + ); +} - const innerVars = collectQueryVariables(inner); - if (innerVars.length === 0) { - // Var-less WHERE body (all-constant triples, e.g. `SELECT * WHERE { - // }`). We cannot hide the sentinel here: a sub-SELECT projecting - // the empty user-variable set is not legal SPARQL, and the bare form would - // let `SELECT *` project the injected `?__dkgViewGraph` (leaking the SWM - // graph IRI, which embeds a wallet address). Decline so the caller falls - // back to the union / per-graph path, which binds no graph variable. Such - // a query over many graphs is not a real view workload, so the fallback's - // cost is irrelevant. - return null; +/** + * The mirror anti-join is valid only for a flat basic graph pattern (plus + * FILTER expressions). Nested graph patterns can differ by boundness, where a + * correlated NOT EXISTS compatibility check is not exact mapping equality. + */ +function isDedupSafeBasicGraphPattern(inner: string): boolean { + const forbidden = ['OPTIONAL', 'MINUS', 'SERVICE', 'VALUES', 'BIND', 'SELECT', 'GRAPH', 'EXISTS']; + let i = 0; + while (i < inner.length) { + const ch = inner[i]; + if (ch === '#') { + while (i < inner.length && inner[i] !== '\n') i++; + continue; + } + if (ch === '"' || ch === "'") { + i = skipSparqlStringLiteral(inner, i); + continue; + } + if (ch === '<') { + const end = skipSparqlIriRef(inner, i); + i = end ?? i + 1; + continue; + } + if (ch === '{' || ch === '}') return false; + if (isKeywordStart(inner, i)) { + let end = i + 1; + while (end < inner.length && isWordContinuation(inner[end])) end++; + if (forbidden.some((keyword) => isSparqlKeyword(inner, i, end, keyword))) { + return false; + } + i = end; + continue; + } + i++; } - - const values = graphUris.map((g) => `<${assertSafeIri(g)}>`).join(' '); - const graphBlock = `VALUES ${VIEW_GRAPH_SENTINEL} { ${values} } GRAPH ${VIEW_GRAPH_SENTINEL} { ${inner} }`; - // Hide the sentinel behind a sub-SELECT that re-exposes only the user's - // variables, so `SELECT *` and cross-graph DISTINCT behave as they did under - // the UNION form. - return `${before} { SELECT ${innerVars.join(' ')} WHERE { ${graphBlock} } } ${after}`; + return true; } /** diff --git a/packages/query/test/read-both-dedup.test.ts b/packages/query/test/read-both-dedup.test.ts new file mode 100644 index 0000000000..3707913593 --- /dev/null +++ b/packages/query/test/read-both-dedup.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; +import { OxigraphStore, type Quad } from '@origintrail-official/dkg-storage'; +import { DKGQueryEngine } from '../src/dkg-query-engine.js'; + +const CG = 'finalization-chain-e2e'; +const ROOT = `did:dkg:context-graph:${CG}`; +const PER_CGID = `${ROOT}/context/7`; +const VM = `${ROOT}/_verifiable_memory/0xAA/1`; +const ENTITY = 'urn:finalization-chain:entity:1'; +const NAME = 'http://schema.org/name'; +const TYPE = 'http://schema.org/additionalType'; + +function q(subject: string, predicate: string, object: string, graph: string): Quad { + return { subject, predicate, object, graph }; +} + +const NAME_QUERY = `SELECT ?name WHERE { <${ENTITY}> <${NAME}> ?name }`; + +describe('verifiable-memory read-both deduplicates mirrored triples (#1270)', () => { + it('collapses an identical triple mirrored between root and per-KA VM', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const triple = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([triple, { ...triple, graph: VM }]); + + const result = await engine.query(NAME_QUERY, { contextGraphId: CG }); + + expect(result.bindings).toEqual([{ name: '"Finalization Chain Draft"' }]); + }); + + it('preserves SELECT bag multiplicity for distinct triples with the same projection', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const name = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([ + name, + q(ENTITY, TYPE, '"Document"', ROOT), + { ...name, graph: VM }, + ]); + + const result = await engine.query( + `SELECT ?s WHERE { ?s ?p ?o . FILTER(?s = <${ENTITY}>) }`, + { contextGraphId: CG }, + ); + + expect(result.bindings).toEqual([{ s: ENTITY }, { s: ENTITY }]); + }); + + it('preserves mappings that differ by an OPTIONAL binding in a later VM graph', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const name = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([ + name, + { ...name, graph: VM }, + q(ENTITY, TYPE, '"Document"', VM), + ]); + + const result = await engine.query( + `SELECT ?name ?type WHERE { + <${ENTITY}> <${NAME}> ?name . + OPTIONAL { <${ENTITY}> <${TYPE}> ?type } + }`, + { contextGraphId: CG, view: 'verifiable-memory' }, + ); + + expect(result.bindings).toHaveLength(2); + expect(result.bindings.filter((binding) => binding['type'] === undefined)).toHaveLength(1); + expect(result.bindings.filter((binding) => binding['type'] === '"Document"')).toHaveLength(1); + }); + + it('actually reads and deduplicates the per-cgId graph in VM view routing', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const triple = q(ENTITY, NAME, '"Finalization Chain Draft"', ROOT); + await store.insert([triple, { ...triple, graph: PER_CGID }]); + + const result = await engine.query(NAME_QUERY, { + contextGraphId: CG, + view: 'verifiable-memory', + }); + + expect(result.bindings).toEqual([{ name: '"Finalization Chain Draft"' }]); + }); + + it('deduplicates a per-cgId and per-KA mirror when the root graph is empty', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + const triple = q(ENTITY, NAME, '"Finalization Chain Draft"', PER_CGID); + await store.insert([triple, { ...triple, graph: VM }]); + + const result = await engine.query(NAME_QUERY, { + contextGraphId: CG, + view: 'verifiable-memory', + }); + + expect(result.bindings).toEqual([{ name: '"Finalization Chain Draft"' }]); + }); +}); From d0884e3096de2f90a78bd26485657e92e0bf4e65 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:57:55 +0200 Subject: [PATCH 04/12] fix(agent): keep outbox retries on schedule (#1579) (#1622) * fix(agent): keep outbox retries on schedule * test(agent): pin scheduled-only reconnect behavior * test(devnet): cover scheduled-only outbox retries * refactor(outbox): make scheduled-only retries canonical * test(outbox): preserve snapshot isolation through list --------- Co-authored-by: Branimir Rakic --- packages/agent/src/dkg-agent-base.ts | 5 +- packages/agent/src/dkg-agent-constants.ts | 21 ++---- packages/agent/src/dkg-agent-join.ts | 27 +++---- packages/agent/src/dkg-agent-lifecycle.ts | 33 +-------- packages/agent/src/p2p/messenger.ts | 28 ++----- packages/agent/test/p2p-resilience.test.ts | 55 +++++++------- packages/core/src/messenger-types.ts | 14 +--- packages/core/src/protocol-outbox.ts | 34 +++------ packages/core/test/protocol-outbox.test.ts | 12 +-- packages/node-ui/src/db.ts | 19 +---- .../node-ui/test/messenger-stores.test.ts | 11 +-- .../devnet-test-issue-1579-outbox-backoff.sh | 74 +++++++++++++++++++ 12 files changed, 153 insertions(+), 180 deletions(-) create mode 100755 scripts/devnet-test-issue-1579-outbox-backoff.sh diff --git a/packages/agent/src/dkg-agent-base.ts b/packages/agent/src/dkg-agent-base.ts index 7fc76ac7d9..5dc4836716 100644 --- a/packages/agent/src/dkg-agent-base.ts +++ b/packages/agent/src/dkg-agent-base.ts @@ -891,9 +891,8 @@ export class DKGAgentBase { protected hostModeReconcilerTimer: ReturnType | null = null; protected hostModePruneTimer: ReturnType | null = null; // rc.9 PR-10: joinApprovalRetryQueue + joinApprovalRetryTimer - // deleted. The substrate's SQLite-backed ProtocolOutbox + its tick - // (`Messenger.processOutboxTick`) + opportunistic on-connect flush - // (`Messenger.processOutboxOnConnect`) replace the entire in-memory + // deleted. The substrate's SQLite-backed ProtocolOutbox + its + // scheduled tick (`Messenger.processOutboxTick`) replace the in-memory // queue: persistence across restart, generic per-protocol coverage, // identical backoff-ladder semantics. Operator-facing diagnostics // (`listPendingJoinApprovalRetries`) are stubbed to [] until PR-12 diff --git a/packages/agent/src/dkg-agent-constants.ts b/packages/agent/src/dkg-agent-constants.ts index 1744d36d05..d19eeba51b 100644 --- a/packages/agent/src/dkg-agent-constants.ts +++ b/packages/agent/src/dkg-agent-constants.ts @@ -172,27 +172,18 @@ export const STORAGE_ACK_REGISTRATION_RETRY_MS = 30_000; * the local curator state is correct but the invitee never learns to * sync, and their own retries can't help because they don't yet hold the * delegation that would let private-sync auth succeed. The tick walks the - * queue's `due()` entries with exponential backoff. Opportunistic retries - * also fire from `connection:open` when the invitee's peer reconnects, - * which usually wins the race; the timer is the safety net for cases - * where reconnect events are missed (e.g. relayed reconnects that don't - * surface a fresh `connection:open` on the curator). + * queue's `due()` entries with exponential backoff. This separate join queue + * retains its own peer lifecycle policy; Universal Messenger rows below are + * scheduled-only. */ export const JOIN_APPROVAL_RETRY_TICK_MS = 30_000; /** * Tick interval for the chat outbox retry queue. Same 30s cadence as * the join-approval queue (`JOIN_APPROVAL_RETRY_TICK_MS`). The cadence - * doesn't gate the FIRST retry — a backoff-due entry that's been - * waiting since 5s after first failure may sit idle for up to 25s - * before this tick picks it up — but the dominant retry trigger in - * practice is the `connection:open` opportunistic flush - * (`processMessageOutboxOnConnect`), which fires the moment the - * recipient peer becomes reachable again. The tick is the safety net - * for cases where reconnect events are missed (e.g. relayed reconnects - * that don't surface a fresh `connection:open` on the sender) or - * where the recipient was reachable all along but transport failures - * are coming from somewhere upstream of libp2p. + * is the sole automatic trigger: reconnect churn must not bypass a row's + * persisted `nextAttemptAt`. A due entry may sit for up to one tick interval + * before the scheduler picks it up. */ export const MESSAGE_OUTBOX_TICK_MS = 30_000; diff --git a/packages/agent/src/dkg-agent-join.ts b/packages/agent/src/dkg-agent-join.ts index 4d81fc168c..eedc4630b2 100644 --- a/packages/agent/src/dkg-agent-join.ts +++ b/packages/agent/src/dkg-agent-join.ts @@ -716,8 +716,7 @@ export class JoinRequestMethods extends DKGAgentBase { ctx, `join-approval for "${contextGraphId}" → ${agentAddress} not delivered now ` + `(error=${result.error ?? 'unknown'}). Curator-local state is correct; ` + - `substrate outbox holds the queued send and will retry on its backoff ` + - `ladder + on the invitee's next reconnect.`, + `substrate outbox holds the queued send and will retry on its backoff ladder.`, ); } @@ -727,8 +726,8 @@ export class JoinRequestMethods extends DKGAgentBase { * delivery state matters. * * Used by: - * * The substrate's periodic outbox tick + on-connect flush — - * both transparent to this call (rc.9 PR-10). + * * The substrate's periodic outbox tick, transparent to this call + * (rc.9 PR-10). * * The operator-facing route `POST /api/context-graph/{id}/redeliver-approval`, * which lets an operator (or peer agent via the chat MCP) re-poke * the curator when the automated retry isn't fast enough. @@ -856,17 +855,16 @@ export class JoinRequestMethods extends DKGAgentBase { */ // rc.9 PR-10: processJoinApprovalRetryQueueTick + // processJoinApprovalRetryQueueOnConnect deleted. The substrate's - // Messenger.processOutboxTick + Messenger.processOutboxOnConnect - // cover /dkg/10.0.1/join-request automatically (same as chat in + // Messenger.processOutboxTick covers /dkg/10.0.1/join-request + // automatically (same as chat in // PR-3), so the two dedicated processors are obsolete. Operator // re-fire route POST /api/context-graph/{id}/redeliver-approval is // unchanged — it still calls redeliverJoinApproval which now // simply re-issues the substrate send. /** - * Re-attempt delivery of a single chat outbox entry. Centralised so - * the periodic tick + the connection:open opportunistic flush share - * one code path. Returns the entry's current state so the caller can + * Re-attempt delivery of a single chat outbox entry from the periodic + * scheduler. Returns the entry's current state so the caller can * decide what to log. * * Goes through `messageHandler.sendChat` directly (bypassing @@ -884,17 +882,14 @@ export class JoinRequestMethods extends DKGAgentBase { * class: an inbound circuit connection from P was open and live, but * every `libp2p.dialProtocol(P, ...)` retry on our side failed with * "The dial request has no valid addresses for peer" for several - * minutes. Daemon logs showed 31 `connection:open` events from P - * (all inbound, all via R) + 20 opportunistic-flush attempts, all - * failing dialProtocol — and then the moment ONE outbound connection - * succeeded (which populated peerStore from outbound identify), the - * very next opportunistic-flush delivered the queued message. + * minutes. Reverse-path enrichment ensures the next scheduled retry sees a + * usable address without letting connection churn bypass persisted backoff. * * The clean fix would be inside libp2p (`dialProtocol` should reuse * an existing open connection of any direction — see PR 5 in the * postmortem follow-up plan), but until that lands, populating * peerStore from the inbound circuit's address gives the dialer - * something to find on the very next attempt. + * something to find on the next scheduled attempt. * * Public so a unit test can exercise it directly without standing up * a full libp2p network (the listener that calls it is registered @@ -1140,7 +1135,7 @@ export class JoinRequestMethods extends DKGAgentBase { ctx, `${label} for "${contextGraphId}" to ${agentAddress} (${targetPeerId}) ` + `queued in substrate outbox: ${sendResult.error}. ` + - `Substrate will retry on its own backoff ladder + on the invitee's next reconnect.`, + `Substrate will retry on its persisted backoff schedule.`, ); return { delivered: false, peerId: targetPeerId, error: sendResult.error }; } diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index ee3dd57f7d..8a58fafda7 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -2345,14 +2345,8 @@ export class LifecycleSyncMethods extends DKGAgentBase { this.node.libp2p.addEventListener('connection:open', (evt) => { const remotePeer = evt.detail.remotePeer.toString(); if (remotePeer === this.node.libp2p.peerId.toString()) return; - // rc.9 PR-10: the dedicated join-approval on-connect flush is - // gone. The substrate's `Messenger.processOutboxOnConnect` (a - // few lines further down in this handler) now covers join- - // approved retries too, since /dkg/10.0.1/join-request is now - // a substrate-managed protocol. - // Reverse-path peerStore enrichment for inbound circuit-relay - // connections, then the symmetric chat-outbox flush. + // connections. // // Closes the "Window D" class from the May 2026 Miles↔Lex 6h // soak postmortem: an inbound circuit connection from peer P @@ -2365,17 +2359,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { // dialProtocol find an address and try it. // // User review on PR #536 caught the original ordering bug: - // running enrichment and the outbox flush in parallel - // fire-and-forget meant the first flush attempt could - // still hit `dialProtocol` against an EMPTY peerStore and - // fail with the same "no valid addresses" error this PR is - // meant to heal — pushing recovery onto the next 30s tick - // or another reconnect. Sequence the two: await enrichment - // first, then flush. Both stay wrapped in their own - // try/catch so an enrichment failure logs a warning and - // still lets the outbox flush proceed (it might succeed - // anyway via a stale-but-usable cached path). - // // The whole chain runs as a fire-and-forget IIFE so the // listener itself doesn't await — libp2p's // `connection:open` emitter is synchronous and we don't @@ -2396,17 +2379,6 @@ export class LifecycleSyncMethods extends DKGAgentBase { const message = err instanceof Error ? err.message : String(err); this.log.warn(ctx, `Reverse-path peerStore enrichment failed for ${remotePeer}: ${message}`); } - // Universal Messenger substrate (rc.9 PR-2/PR-3): drain - // the generic outbox for this peer. Replaces the rc.8 - // chat-specific outbox flush — the substrate now carries - // chat (PR-3) and will carry every other short-message - // protocol after PR-8..PR-11. - try { - await this.messenger.processOutboxOnConnect(remotePeer); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - this.log.warn(ctx, `Opportunistic Messenger-outbox retry on connect failed for ${remotePeer}: ${message}`); - } // PR-2 (SWM-fanout plan): drain pending sender-key packages // that were queued because the recipient had no advertised // peerId at publish time. Tolerant of profile-lookup failure @@ -2604,8 +2576,7 @@ export class LifecycleSyncMethods extends DKGAgentBase { // MESSAGE_OUTBOX_TICK_MS for the rationale (silent-drop on // transport failure used to lose operator-typed messages from // `dkg_send_message`; this is the safety-net retry loop that turns - // them into eventual successes, complemented by the - // opportunistic-on-reconnect path in the connection:open listener). + // them into eventual successes on their persisted retry schedule. // Universal Messenger substrate retry tick (rc.9 PR-2 + // PR-3). The rc.8 chat-specific tick was deleted in PR-3; // this is now the only outbox tick — chat (PR-3) and every diff --git a/packages/agent/src/p2p/messenger.ts b/packages/agent/src/p2p/messenger.ts index 812c3e9e4f..0dd9711d69 100644 --- a/packages/agent/src/p2p/messenger.ts +++ b/packages/agent/src/p2p/messenger.ts @@ -633,8 +633,8 @@ export class Messenger { // Inflight guard (rc.9 #521 lesson lifted): two parallel // attempters on the same `(peer, protocol, messageId)` can race - // when the periodic tick + an opportunistic-flush fire close - // together. Second attempter exits without dialing. + // when a first sender + periodic retry (or overlapping explicit callers) + // race. Second attempter exits without dialing. if (!outbox.tryBeginAttempt(peerId, protocolId, messageId)) { // Another attempt is in flight. This is not the same thing as // durable queued: the winning attempt may still be on its first @@ -798,24 +798,6 @@ export class Messenger { } } - /** - * Opportunistic-flush retry loop. The lifecycle.ts wiring (PR-3) - * calls this from a libp2p `connection:open` event for `peerId`: - * a reconnection is the signal we were waiting for, so attempt - * every pending entry for `peer` NOW even if `nextAttemptAt` is - * still in the future. - * - * Same guards as `processOutboxTick` — must check `hasEntry` - * after `tryBeginAttempt` to defend against the rc.9 #538 race. - */ - async processOutboxOnConnect(peerId: string): Promise { - if (!this.outbox) return; - const pending = this.outbox.pendingFor(peerId); - for (const entry of pending) { - await this.retryOutboxEntry(entry); - } - } - private async retryOutboxEntry(entry: { peer: string; protocol: string; @@ -827,7 +809,7 @@ export class Messenger { return; } try { - // Stale-snapshot guard — between the moment `due`/`pendingFor` + // Stale-snapshot guard — between the moment `due` // gave us the snapshot and the moment `tryBeginAttempt` won, // a sibling flush may have completed delivery and called // `markDelivered`. Re-check `hasEntry` and bail if gone. @@ -879,7 +861,7 @@ export class Messenger { * * Fire-and-forget: never blocks the caller. The DHT walk's * side-effect (populating `peerStore` for the peer) heals the - * next opportunistic-flush or periodic-tick retry, not the + * next periodic-tick retry, not the * current one. This is intentional — the current retry has * already failed; the walk is for the next attempt. * @@ -923,7 +905,7 @@ export class Messenger { } private clearDhtWalkRateLimitIfDrained(peerId: string): void { - if (!this.outbox || this.outbox.pendingFor(peerId).length === 0) { + if (!this.outbox || !this.outbox.hasPendingFor(peerId)) { this.lastDhtWalkAt.delete(peerId); } } diff --git a/packages/agent/test/p2p-resilience.test.ts b/packages/agent/test/p2p-resilience.test.ts index 2976508b34..9499586d04 100644 --- a/packages/agent/test/p2p-resilience.test.ts +++ b/packages/agent/test/p2p-resilience.test.ts @@ -338,21 +338,7 @@ describe('p2p resilience hooks', () => { } }, 15_000); - // User review on PR #536: the `connection:open` listener used to - // call `enrichPeerStoreFromInboundCircuit` and the outbox flush - // in parallel fire-and-forget, which meant the first outbox - // flush attempt could still hit `dialProtocol` against an EMPTY - // peerStore — exact same "no valid addresses for peer" failure - // the PR is meant to heal. Fix wraps both in a sequential IIFE - // so the flush sees the freshly-merged reverse-path address. - // - // rc.9 PR-3 substrate cutover: the outbox flush moved from - // `processMessageOutboxOnConnect` (chat-specific, deleted) to - // `messenger.processOutboxOnConnect` (substrate, generic). The - // ordering invariant is the same — enrich BEFORE the substrate - // flush so the substrate's dialProtocol calls see the merged - // peerStore address. - it('awaits enrichment BEFORE the substrate outbox flush on inbound circuit open (PR #536 review, rc.9 substrate)', async () => { + it('enriches reverse paths without waking the Messenger outbox on connection open', async () => { const agent = await DKGAgent.create({ name: 'ReversePathEnrichBeforeFlush', listenHost: '127.0.0.1', @@ -363,8 +349,25 @@ describe('p2p resilience hooks', () => { allowAllNetworkAdmission(agent); const events: string[] = []; + const wireProtocols: string[] = []; let enrichResolve: (() => void) | null = null; const enrichDone = new Promise((r) => { enrichResolve = r; }); + const remotePeer = freshPeerIdString(); + const testProtocol = '/dkg/test/scheduled-only'; + const queuedAt = Date.now(); + const messenger = (agent as any).messenger; + messenger.outbox.enqueueFailure( + remotePeer, + testProtocol, + 'scheduled-only-message', + new Uint8Array([1, 2, 3]), + 'offline', + queuedAt, + ); + messenger.router.send = async (_peer: string, protocol: string) => { + wireProtocols.push(protocol); + return new Uint8Array([0x42]); + }; (agent as any).enrichPeerStoreFromInboundCircuit = async () => { events.push('enrich:start'); @@ -372,11 +375,6 @@ describe('p2p resilience hooks', () => { events.push('enrich:end'); enrichResolve?.(); }; - (agent as any).messenger.processOutboxOnConnect = async () => { - events.push('flush:start'); - }; - - const remotePeer = freshPeerIdString(); const relayPeer = freshPeerIdString(); agent.node.libp2p.dispatchEvent(new CustomEvent('connection:open', { detail: { @@ -391,23 +389,22 @@ describe('p2p resilience hooks', () => { } as any)); await enrichDone; - // Give the IIFE one more tick to schedule the flush. + // Give the connection lifecycle's remaining async work time to settle. await new Promise(r => setTimeout(r, 50)); expect(events).toContain('enrich:start'); expect(events).toContain('enrich:end'); - expect(events).toContain('flush:start'); - expect(events.indexOf('enrich:end')).toBeLessThan(events.indexOf('flush:start')); + expect(wireProtocols).not.toContain(testProtocol); + + // The same row remains deliverable by the sole automatic trigger once + // its persisted backoff has elapsed. + await messenger.processOutboxTick(queuedAt + 10_000); + expect(wireProtocols.filter((protocol) => protocol === testProtocol)).toHaveLength(1); + expect(messenger.outbox.hasPendingFor(remotePeer)).toBe(false); } finally { await agent.stop().catch(() => {}); } }, 15_000); - // Stale-snapshot guard regression: the equivalent test for the - // substrate's `Messenger.processOutboxOnConnect` lives in - // `messenger-substrate.test.ts` ("honours the stale-snapshot - // guard (rc.9 #538)") — same race, same fix, just at the - // generic substrate layer rather than the chat-specific one - // which was deleted in rc.9 PR-3. }); }); diff --git a/packages/core/src/messenger-types.ts b/packages/core/src/messenger-types.ts index bb82654b5e..f292c887ad 100644 --- a/packages/core/src/messenger-types.ts +++ b/packages/core/src/messenger-types.ts @@ -197,22 +197,16 @@ export interface ProtocolOutboxStore { /** * Whether an entry exists for `(peer, protocol, messageId)`. Used - * by the stale-snapshot guard in `Messenger.processOutboxOnConnect` - * — between `tryBeginAttempt` (inflight lock) and the wire send, + * by the scheduled drain's stale-snapshot guard — between + * `tryBeginAttempt` (inflight lock) and the wire send, * a sibling flush may have already delivered + removed the entry, * and we must not double-send. The rc9 #538 fix lifted into the * generic substrate. */ hasEntry(peer: string, protocol: string, messageId: string): boolean; - /** - * All entries for a specific peer, regardless of `nextAttemptAt`. - * Used by `processOutboxOnConnect`: a reconnection is the signal - * we were waiting for, so attempt now even if backoff isn't due - * yet. Sorted by `firstFailureAt` ascending for FIFO per-peer - * drain. - */ - pendingFor(peer: string): ProtocolOutboxEntry[]; + /** Whether this peer still has any durable row (DHT recovery bookkeeping). */ + hasPendingFor(peer: string): boolean; /** * All entries whose `nextAttemptAt <= now`. Used by the periodic diff --git a/packages/core/src/protocol-outbox.ts b/packages/core/src/protocol-outbox.ts index 92351b8db6..53e55f5726 100644 --- a/packages/core/src/protocol-outbox.ts +++ b/packages/core/src/protocol-outbox.ts @@ -102,17 +102,9 @@ export class ProtocolOutbox { private readonly backoffs: readonly number[]; /** * Per-key inflight set to prevent concurrent retry attempts for the - * same `(peer, protocol, messageId)`. Two trigger surfaces — the - * periodic tick (`Messenger.processOutboxTick`) and the - * opportunistic flush on `connection:open` - * (`Messenger.processOutboxOnConnect`) — can interleave: the tick - * starts the send for entry E, JS yields, `connection:open` fires, - * the on-connect handler reads `pendingFor(peer)` (entry E is still - * there — `markDelivered` hasn't fired yet because the in-flight - * send hasn't resolved), and would start a CONCURRENT second send - * for the same entry. Worst case both succeed and the receiver sees - * the same payload twice (receiver dedup absorbs it, but we waste - * a round-trip and amplify load). + * same `(peer, protocol, messageId)`. Overlapping scheduler callers or + * another explicit sender can otherwise interleave around a stale due + * snapshot and duplicate the same wire attempt. * * `tryBeginAttempt` is an atomic check-and-set: the second * concurrent attempter sees `false` and exits without dialing. @@ -206,13 +198,8 @@ export class ProtocolOutbox { return this.store.due(now); } - /** - * All entries for `peer`, regardless of `nextAttemptAt`. Used by - * `Messenger.processOutboxOnConnect` for opportunistic flush on - * reconnection. - */ - pendingFor(peer: string): ProtocolOutboxEntry[] { - return this.store.pendingFor(peer); + hasPendingFor(peer: string): boolean { + return this.store.hasPendingFor(peer); } /** Drop entries older than the store's configured max-age. */ @@ -327,11 +314,8 @@ export class InMemoryProtocolOutboxStore implements ProtocolOutboxStore { return this.entries.has(InMemoryProtocolOutboxStore.key(peer, protocol, messageId)); } - pendingFor(peer: string): ProtocolOutboxEntry[] { - return Array.from(this.entries.values()) - .filter((e) => e.peer === peer) - .sort((a, b) => a.firstFailureAt - b.firstFailureAt) - .map(cloneOutboxEntry); + hasPendingFor(peer: string): boolean { + return Array.from(this.entries.values()).some((entry) => entry.peer === peer); } due(now: number): ProtocolOutboxEntry[] { @@ -356,12 +340,12 @@ export class InMemoryProtocolOutboxStore implements ProtocolOutboxStore { } list(): ProtocolOutboxEntry[] { - return Array.from(this.entries.values()).map((e) => ({ ...e })); + return Array.from(this.entries.values()).map(cloneOutboxEntry); } getEntry(peer: string, protocol: string, messageId: string): ProtocolOutboxEntry | undefined { const entry = this.entries.get(InMemoryProtocolOutboxStore.key(peer, protocol, messageId)); - return entry ? { ...entry } : undefined; + return entry ? cloneOutboxEntry(entry) : undefined; } } diff --git a/packages/core/test/protocol-outbox.test.ts b/packages/core/test/protocol-outbox.test.ts index 110dbb193f..ffad293ba5 100644 --- a/packages/core/test/protocol-outbox.test.ts +++ b/packages/core/test/protocol-outbox.test.ts @@ -71,7 +71,7 @@ describe('ProtocolOutbox.enqueueFailure', () => { payload[0] = 9; entry.payload[1] = 8; - const pending = outbox.pendingFor(PEER_A); + const pending = outbox.list().filter((entry) => entry.peer === PEER_A); expect(Array.from(pending[0].payload)).toEqual([1, 2, 3]); pending[0].payload[2] = 7; @@ -128,7 +128,7 @@ describe('ProtocolOutbox.tryBeginAttempt / endAttempt', () => { }); }); -describe('ProtocolOutbox.due / pendingFor', () => { +describe('ProtocolOutbox.due / peer presence', () => { it('due returns entries whose nextAttemptAt is at or before now', () => { const { outbox } = fixture(); outbox.enqueueFailure(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1000); @@ -137,14 +137,14 @@ describe('ProtocolOutbox.due / pendingFor', () => { expect(outbox.due(expectedNext)).toHaveLength(1); }); - it('pendingFor returns all entries for a peer in firstFailureAt ascending order', () => { + it('hasPendingFor tracks peer rows without exposing a reconnect drain snapshot', () => { const { outbox } = fixture(); outbox.enqueueFailure(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2000); outbox.enqueueFailure(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1000); outbox.enqueueFailure(PEER_B, PROTO, MSG_1, PAYLOAD, 'e', 500); - const peerA = outbox.pendingFor(PEER_A); - expect(peerA.map((e) => e.messageId)).toEqual([MSG_1, MSG_2]); - expect(outbox.pendingFor(PEER_B)).toHaveLength(1); + expect(outbox.hasPendingFor(PEER_A)).toBe(true); + expect(outbox.hasPendingFor(PEER_B)).toBe(true); + expect(outbox.hasPendingFor('peer-c')).toBe(false); }); }); diff --git a/packages/node-ui/src/db.ts b/packages/node-ui/src/db.ts index 645cae4059..5eb161754e 100644 --- a/packages/node-ui/src/db.ts +++ b/packages/node-ui/src/db.ts @@ -2774,23 +2774,8 @@ export class SqliteProtocolOutboxStore implements ProtocolOutboxStore { return row !== undefined; } - pendingFor(peer: string): ProtocolOutboxEntry[] { - const rows = this.db - .prepare( - `SELECT * FROM protocol_outbox WHERE peer_id = ? ORDER BY first_failure_at ASC`, - ) - .all(peer) as Array<{ - peer_id: string; - protocol: string; - message_id: string; - payload: Buffer; - attempts: number; - first_failure_at: number; - last_attempt_at: number; - next_attempt_at: number; - last_error: string | null; - }>; - return rows.map(SqliteProtocolOutboxStore.rowToEntry); + hasPendingFor(peer: string): boolean { + return this.db.prepare('SELECT 1 FROM protocol_outbox WHERE peer_id = ? LIMIT 1').get(peer) !== undefined; } due(now: number): ProtocolOutboxEntry[] { diff --git a/packages/node-ui/test/messenger-stores.test.ts b/packages/node-ui/test/messenger-stores.test.ts index 21efc6e152..b88f8bf619 100644 --- a/packages/node-ui/test/messenger-stores.test.ts +++ b/packages/node-ui/test/messenger-stores.test.ts @@ -180,7 +180,7 @@ describe('SqliteProtocolOutboxStore', () => { payload[0] = 9; entry.payload[1] = 8; - const pending = store.pendingFor(PEER_A); + const pending = store.list().filter((entry) => entry.peer === PEER_A); expect(Array.from(pending[0].payload)).toEqual([1, 2, 3]); pending[0].payload[2] = 7; @@ -207,13 +207,14 @@ describe('SqliteProtocolOutboxStore', () => { expect(store.markDelivered(PEER_A, PROTO, MSG_1)).toBe(false); }); - it('pendingFor returns entries sorted by firstFailureAt ascending', () => { + it('hasPendingFor reports whether a peer owns any durable row', () => { const store = new SqliteProtocolOutboxStore(db); store.enqueue(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2000); store.enqueue(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1000); store.enqueue(PEER_B, PROTO, MSG_1, PAYLOAD, 'e', 500); - expect(store.pendingFor(PEER_A).map((e) => e.messageId)).toEqual([MSG_1, MSG_2]); - expect(store.pendingFor(PEER_B)).toHaveLength(1); + expect(store.hasPendingFor(PEER_A)).toBe(true); + expect(store.hasPendingFor(PEER_B)).toBe(true); + expect(store.hasPendingFor('peer-c')).toBe(false); }); it('due returns entries with nextAttemptAt <= now', () => { @@ -259,7 +260,7 @@ describe('SqliteProtocolOutboxStore', () => { db.close(); db = new DashboardDB({ dataDir: dir }); const reopened = new SqliteProtocolOutboxStore(db); - const pending = reopened.pendingFor(PEER_A); + const pending = reopened.list().filter((entry) => entry.peer === PEER_A); expect(pending).toHaveLength(1); expect(pending[0].lastError).toBe('crash-before-delivery'); expect(Array.from(pending[0].payload)).toEqual(Array.from(PAYLOAD)); diff --git a/scripts/devnet-test-issue-1579-outbox-backoff.sh b/scripts/devnet-test-issue-1579-outbox-backoff.sh new file mode 100755 index 0000000000..98ec0e1ac4 --- /dev/null +++ b/scripts/devnet-test-issue-1579-outbox-backoff.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1579. A cooling-down reliable message is seeded +# into node1's durable outbox, node2 is restarted/reconnected, and the row must +# remain untouched until its scheduled next_attempt_at. On the buggy build the +# connection:open hook immediately sends/removes (or advances) the row. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +DB="$DEVNET_DIR/node1/node-ui.db" +MESSAGE_ID="devnet-issue-1579-$(date +%s)" + +fail() { echo "[#1579] FAIL: $*" >&2; exit 1; } +cleanup() { + DB="$DB" MESSAGE_ID="$MESSAGE_ID" node --input-type=module <<'NODE' >/dev/null 2>&1 || true +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB); +db.prepare('DELETE FROM protocol_outbox WHERE message_id = ?').run(process.env.MESSAGE_ID); +db.close(); +NODE +} +trap cleanup EXIT + +[[ -f "$DB" ]] || fail "node1 database missing; start a devnet first" +. "$ROOT/scripts/devnet-lib.sh" + +status2="$(body_of "$(api 2 GET /api/status)")" +peer2="$(field "$status2" peerId)" +[[ -n "$peer2" ]] || fail "node2 peerId unavailable" + +DB="$DB" PEER="$peer2" MESSAGE_ID="$MESSAGE_ID" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +import { encodeReliableEnvelope, PROTOCOL_MESSAGE } from './packages/core/dist/index.js'; +const now = Date.now(); +const payload = encodeReliableEnvelope({ + messageId: process.env.MESSAGE_ID, + version: 1, + tsMs: now, + payload: new TextEncoder().encode('{"type":"chat","text":"#1579 probe"}'), +}); +const db = new Database(process.env.DB); +db.prepare(`INSERT INTO protocol_outbox + (peer_id, protocol, message_id, payload, attempts, first_failure_at, + last_attempt_at, next_attempt_at, last_error) + VALUES (?, ?, ?, ?, 1, ?, ?, ?, ?)`) + .run(process.env.PEER, PROTOCOL_MESSAGE, process.env.MESSAGE_ID, + Buffer.from(payload), now, now, now + 60 * 60 * 1000, 'devnet probe'); +db.close(); +NODE + +"$ROOT/scripts/devnet.sh" restart-node 2 >/dev/null +for _ in $(seq 1 60); do + [[ "$(code_of "$(api 2 GET /api/status)")" == 200 ]] && break + sleep 1 +done +[[ "$(code_of "$(api 2 GET /api/status)")" == 200 ]] || fail "node2 did not restart" + +connect="$(api 1 POST /api/connect "{\"peerId\":\"$peer2\"}")" +[[ "$(code_of "$connect")" == 200 ]] || fail "node1 could not reconnect to node2" +sleep 5 + +row="$(DB="$DB" MESSAGE_ID="$MESSAGE_ID" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare('SELECT attempts, next_attempt_at FROM protocol_outbox WHERE message_id = ?').get(process.env.MESSAGE_ID); +process.stdout.write(row ? JSON.stringify(row) : 'missing'); +db.close(); +NODE +)" +[[ "$row" != missing ]] || fail "cooling row was drained on connection-open" +attempts="$(field "$row" attempts)" +[[ "$attempts" == 1 ]] || fail "connection-open advanced attempts to $attempts" +echo "[#1579] PASS: reconnect preserved the cooling-down outbox row" From edc69e0d992d31889bfbe1827e538f0762791086 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:58:34 +0200 Subject: [PATCH 05/12] fix(agent): keep StorageACK retries collector-owned (#1577) (#1624) * fix(agent): keep storage ACK retries collector-owned * fix(agent): clear request-owned SLO state * refactor(agent): make reliable failure policy explicit * test(devnet): cover StorageACK retry ownership * refactor(agent): split request-owned reliable sends --------- Co-authored-by: Branimir Rakic --- packages/agent/src/dkg-agent.ts | 6 +- packages/agent/src/p2p/messenger.ts | 33 +++++++++++ .../agent/test/messenger-substrate.test.ts | 16 ++++++ .../test/v10-ack-provider-wiring.test.ts | 26 ++++----- ...et-test-issue-1577-ack-outbox-ownership.sh | 56 +++++++++++++++++++ 5 files changed, 121 insertions(+), 16 deletions(-) create mode 100755 scripts/devnet-test-issue-1577-ack-outbox-ownership.sh diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 59ed216dc6..f27f95994a 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -618,7 +618,7 @@ function normalizeStorageAckConfig(config: DKGAgentConfig): ResolvedDKGAgentConf } interface ACKReliableMessenger { - sendReliable( + sendRequestOwned( peerId: string, protocol: string, data: Uint8Array, @@ -631,11 +631,11 @@ function createACKSendP2P(input: { timeoutMs: number; }): ACKCollectorDeps['sendP2P'] { return async (peerId: string, protocol: string, data: Uint8Array) => { - const sendResult = await input.messenger.sendReliable(peerId, protocol, data, { + const sendResult = await input.messenger.sendRequestOwned(peerId, protocol, data, { timeoutMs: input.timeoutMs, }); if (!sendResult.delivered) { - throw new Error(`substrate queued (transport): ${sendResult.error}`); + throw new Error(`substrate send already in flight (transport): ${sendResult.error}`); } if (!sendResult.response) { throw new Error('substrate delivered (transport) without response'); diff --git a/packages/agent/src/p2p/messenger.ts b/packages/agent/src/p2p/messenger.ts index 0dd9711d69..bfa08b2474 100644 --- a/packages/agent/src/p2p/messenger.ts +++ b/packages/agent/src/p2p/messenger.ts @@ -250,6 +250,12 @@ export type ReliableSendResult = error: string; }; +/** The explicit throw policy can never produce a durable queued result. */ +export type ThrowingReliableSendResult = Exclude< + ReliableSendResult, + { delivered: false; queued: true } +>; + /** Handler signature for `Messenger.register`. */ export type ReliableHandler = ( payload: Uint8Array, @@ -574,6 +580,26 @@ export class Messenger { protocolId: string, payload: Uint8Array, opts: SendReliableOpts = {}, + ): Promise { + return this.sendFramed(peerId, protocolId, payload, opts, true); + } + + /** Reliable framing/idempotency for bounded request-owned retries; never queues. */ + async sendRequestOwned( + peerId: string, + protocolId: string, + payload: Uint8Array, + opts: SendReliableOpts = {}, + ): Promise { + return this.sendFramed(peerId, protocolId, payload, opts, false) as Promise; + } + + private async sendFramed( + peerId: string, + protocolId: string, + payload: Uint8Array, + opts: SendReliableOpts, + queueRecoverableFailure: boolean, ): Promise { this.requireSubstrate('sendReliable'); @@ -669,6 +695,13 @@ export class Messenger { if (!isRecoverableMessengerSendError(err, errMsg)) { throw err; } + if (!queueRecoverableFailure) { + // No durable row can later deliver or expire this request, so its SLO + // start marker has no future owner. Clear it before returning control + // to the request-scoped retry loop. + this.firstAttemptAt.delete(sloK); + throw err; + } const entry = outbox.enqueueFailure( peerId, protocolId, diff --git a/packages/agent/test/messenger-substrate.test.ts b/packages/agent/test/messenger-substrate.test.ts index c6c9574bda..eef8324774 100644 --- a/packages/agent/test/messenger-substrate.test.ts +++ b/packages/agent/test/messenger-substrate.test.ts @@ -143,6 +143,22 @@ describe('Messenger.sendReliable (happy path semantics)', () => { }); }); +describe('Messenger.sendRequestOwned', () => { + it('keeps reliable framing but does not persist recoverable failures', async () => { + const router = makeRouter(async () => { + throw new Error('no valid addresses for peer'); + }); + const { messenger, outboxStore } = makeSubstrate({ router }); + + await expect(messenger.sendRequestOwned(PEER_A, PROTO, new Uint8Array([1]), { + messageId: FIXED_MSG_ID, + })).rejects.toThrow('no valid addresses for peer'); + expect(outboxStore.size()).toBe(0); + expect(() => decodeReliableEnvelope(router.send.calls[0][2] as Uint8Array)).not.toThrow(); + expect((messenger as any).firstAttemptAt.size).toBe(0); + }); +}); + describe('Messenger.sendReliable (sender-side idempotency)', () => { it('returns the cached response on a second send with the same messageId, no router call', async () => { const router = makeRouter(async () => new Uint8Array([0x42])); diff --git a/packages/agent/test/v10-ack-provider-wiring.test.ts b/packages/agent/test/v10-ack-provider-wiring.test.ts index b89af6aa51..52d7ce549c 100644 --- a/packages/agent/test/v10-ack-provider-wiring.test.ts +++ b/packages/agent/test/v10-ack-provider-wiring.test.ts @@ -268,12 +268,12 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # agent = boot.agent; const internals = boot.internals; const response = new Uint8Array([9]); - const sendReliable = vi.fn(async () => ({ + const sendRequestOwned = vi.fn(async () => ({ delivered: true, response, })); const payload = new Uint8Array([1, 2, 3]); - internals.messenger = { sendReliable }; + internals.messenger = { sendRequestOwned }; internals.createV10ACKProvider('test-cg'); const publishDeps = capturedAckCollectorDeps[0] as ACKCollectorDepsCapture; @@ -283,10 +283,10 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # const updateDeps = capturedAckCollectorDeps[1] as ACKCollectorDepsCapture; await expect(updateDeps.sendP2P!('peer-b', '/dkg/test/storage-update-ack', payload)).resolves.toEqual(response); - expect(sendReliable).toHaveBeenNthCalledWith(1, 'peer-a', '/dkg/test/storage-ack', payload, { + expect(sendRequestOwned).toHaveBeenNthCalledWith(1, 'peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 60_000, }); - expect(sendReliable).toHaveBeenNthCalledWith(2, 'peer-b', '/dkg/test/storage-update-ack', payload, { + expect(sendRequestOwned).toHaveBeenNthCalledWith(2, 'peer-b', '/dkg/test/storage-update-ack', payload, { timeoutMs: 60_000, }); }); @@ -296,18 +296,18 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # agent = boot.agent; const internals = boot.internals; const response = new Uint8Array([9]); - const sendReliable = vi.fn(async () => ({ + const sendRequestOwned = vi.fn(async () => ({ delivered: true, response, })); - internals.messenger = { sendReliable }; + internals.messenger = { sendRequestOwned }; const payload = new Uint8Array([1]); internals.createV10ACKProvider('test-cg'); const publishDeps = capturedAckCollectorDeps[0] as ACKCollectorDepsCapture; await expect(publishDeps.sendP2P!('peer-a', '/dkg/test/storage-ack', payload)).resolves.toEqual(response); - expect(sendReliable).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { + expect(sendRequestOwned).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 60_000, }); }); @@ -340,11 +340,11 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # agent = boot.agent; const internals = boot.internals; const response = new Uint8Array([9]); - const sendReliable = vi.fn(async () => ({ + const sendRequestOwned = vi.fn(async () => ({ delivered: true, response, })); - internals.messenger = { sendReliable }; + internals.messenger = { sendRequestOwned }; const payload = new Uint8Array([1]); internals.createV10ACKProvider('test-cg'); @@ -355,7 +355,7 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # handlerDeadlineMs: 0, sendTimeoutMs: 20_000, }); - expect(sendReliable).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { + expect(sendRequestOwned).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 20_000, }); }); @@ -370,10 +370,10 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # delivered: false, error: 'queued', })); - internals.messenger = { sendReliable: queuedSend }; + internals.messenger = { sendRequestOwned: queuedSend }; await expect( internals.createACKTransportFactory()().sendP2P('peer-a', '/dkg/test/storage-ack', payload), - ).rejects.toThrow(/substrate queued \(transport\): queued/); + ).rejects.toThrow(/substrate send already in flight \(transport\): queued/); expect(queuedSend).toHaveBeenCalledWith('peer-a', '/dkg/test/storage-ack', payload, { timeoutMs: 60_000, }); @@ -381,7 +381,7 @@ describe('DKGAgent.createV10ACKProvider — structured ACK verifier wiring (PR # const missingResponseSend = vi.fn(async () => ({ delivered: true, })); - internals.messenger = { sendReliable: missingResponseSend }; + internals.messenger = { sendRequestOwned: missingResponseSend }; await expect( internals.createACKTransportFactory()().sendP2P('peer-a', '/dkg/test/storage-ack', payload), ).rejects.toThrow(/substrate delivered \(transport\) without response/); diff --git a/scripts/devnet-test-issue-1577-ack-outbox-ownership.sh b/scripts/devnet-test-issue-1577-ack-outbox-ownership.sh new file mode 100755 index 0000000000..240004ea87 --- /dev/null +++ b/scripts/devnet-test-issue-1577-ack-outbox-ownership.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1577. Stop one of four cores, publish from an edge +# (the other three can still satisfy quorum), then prove the finished collector +# left no durable StorageACK request behind on the publisher. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +PUBLISHER="${ACK_OUTBOX_PUBLISHER:-5}" +TARGET="${ACK_OUTBOX_TARGET:-4}" +CG="${DEVNET_CONTEXT_GRAPH:-devnet-test}" +DB="$DEVNET_DIR/node$PUBLISHER/node-ui.db" +stopped=0 + +fail() { echo "[#1577] FAIL: $*" >&2; exit 1; } +cleanup() { + if [[ "$stopped" == 1 ]]; then "$ROOT/scripts/devnet.sh" restart-node "$TARGET" >/dev/null 2>&1 || true; fi +} +trap cleanup EXIT INT TERM +. "$ROOT/scripts/devnet-lib.sh" + +for n in 1 2 3 4 5; do + [[ "$(code_of "$(api "$n" GET /api/status)")" == 200 ]] || fail "node$n is not ready (need 4 cores + edge)" +done +[[ -f "$DB" ]] || fail "publisher database missing: $DB" + +count_ack_rows() { + DB="$DB" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare(`SELECT COUNT(*) AS n FROM protocol_outbox + WHERE protocol IN ('/dkg/10.0.1/storage-ack','/dkg/10.0.2/storage-ack','/dkg/10.0.1/storage-update-ack')`).get(); +process.stdout.write(String(row.n)); db.close(); +NODE +} +baseline="$(count_ack_rows)" + +"$ROOT/scripts/devnet.sh" stop-node "$TARGET" >/dev/null +stopped=1 +sleep 5 + +name="issue-1577-$(date +%s)-$$" +subject="urn:issue:1577:$name" +api "$PUBLISHER" POST /api/knowledge-assets "{\"contextGraphId\":\"$CG\",\"name\":\"$name\"}" >/dev/null +api "$PUBLISHER" POST "/api/knowledge-assets/$name/wm/write" \ + "{\"contextGraphId\":\"$CG\",\"quads\":[{\"subject\":\"$subject\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"ack ownership probe\\\"\",\"graph\":\"\"}]}" >/dev/null +api "$PUBLISHER" POST "/api/knowledge-assets/$name/wm/finalize" "{\"contextGraphId\":\"$CG\"}" >/dev/null +api "$PUBLISHER" POST "/api/knowledge-assets/$name/swm/share" "{\"contextGraphId\":\"$CG\"}" >/dev/null +published="$(api "$PUBLISHER" POST "/api/knowledge-assets/$name/vm/publish" "{\"contextGraphId\":\"$CG\"}")" +[[ "$(code_of "$published")" == 200 ]] || fail "publish did not reach quorum: $(body_of "$published")" +[[ "$(field "$(body_of "$published")" status)" == confirmed ]] || fail "publish was not confirmed" + +rows="$(count_ack_rows)" +[[ "$rows" == "$baseline" ]] || fail "StorageACK outbox rows grew from $baseline to $rows" +echo "[#1577] PASS: collector completed with no durable StorageACK rows" From 058e1da45d496c1a37d7c5e3adfa03727085a184 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:58:55 +0200 Subject: [PATCH 06/12] fix(cli): reject unclaimable async publish jobs (#1576) (#1625) * fix(cli): reject unclaimable async publish jobs * fix(cli): classify publisher readiness recovery * fix(cli): centralize async publisher readiness * test(devnet): cover async publisher readiness gate * test(cli): pin publisher availability lifecycle states * test(cli): provision async publisher in live daemon e2e --------- Co-authored-by: Branimir Rakic --- packages/cli/src/daemon/handle-request.ts | 4 +- packages/cli/src/daemon/lifecycle.ts | 18 ++++- packages/cli/src/daemon/routes/context.ts | 4 +- packages/cli/src/daemon/routes/epcis.ts | 14 +++- .../cli/src/daemon/routes/knowledge-assets.ts | 15 ++++ packages/cli/src/publisher-runner.ts | 44 +++++++++++ packages/cli/test/helpers/live-daemon.ts | 34 ++++++++ ...knowledge-assets-1116-share-errors.test.ts | 78 ++++++++++++++++++- .../cli/test/publisher-availability.test.ts | 39 ++++++++++ .../test/source-worker-daemon-client.test.ts | 2 +- .../cli/test/source-worker-runner.test.ts | 2 +- ...net-test-issue-1576-publisher-readiness.sh | 66 ++++++++++++++++ 12 files changed, 310 insertions(+), 10 deletions(-) create mode 100644 packages/cli/test/publisher-availability.test.ts create mode 100755 scripts/devnet-test-issue-1576-publisher-readiness.sh diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index dbc6f99039..4297ac48c3 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -99,7 +99,7 @@ import { slotEntryPoint, CLI_NPM_PACKAGE, } from '../config.js'; -import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../publisher-runner.js'; +import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type AsyncPublisherAvailability, type PublisherRuntime } from '../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../catchup-runner.js'; import { loadTokens, httpAuthGuard, extractBearerToken } from '../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; @@ -365,6 +365,7 @@ export async function handleRequest( admission: AdmissionStatsView, emitMemoryGraphChanged?: (event: MemoryGraphChangedEvent) => void, emitNotification?: (event: NotificationSseEvent) => void, + publisherAvailability?: AsyncPublisherAvailability, ): Promise { const url = new URL(req.url ?? "/", `http://${req.headers.host}`); const path = url.pathname; @@ -381,6 +382,7 @@ export async function handleRequest( agent, publisherControl, publisherRuntime, + publisherAvailability, config, startedAt, dashDb, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 687f5ebf3b..6aad669619 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -148,7 +148,7 @@ import { import { resolveOtelSignals, resolveLogExporterMode, isUnknownLogExporter } from '../telemetry-config.js'; import { createDaemonLogSink } from './log-sink.js'; import { startRpcUsageTelemetry } from './rpc-usage-log.js'; -import { createPublicSnapshotStore, createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../publisher-runner.js'; +import { createPublicSnapshotStore, createPublisherControlFromStore, resolveAsyncPublisherAvailability, startPublisherRuntimeIfEnabled, type AsyncPublisherAvailability, type PublisherRuntime } from '../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../catchup-runner.js'; import { loadTokens, httpAuthGuard } from '../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; @@ -1717,6 +1717,11 @@ export async function runDaemonInner( }); let publisherRuntime: PublisherRuntime | null = null; + let publisherAvailability: AsyncPublisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime: null, + lifecycleReason: config.publisher?.enabled ? 'publisher_starting' : 'publisher_disabled', + }); // Holds the running async-promote worker lifecycle (PR #3 of the // async-promote-queue series). Initialised in `startPostApiPublishing` // after the API is up so a recoverOnStartup hiccup never blocks boot; @@ -1988,7 +1993,17 @@ export async function runDaemonInner( log, }); publisherRuntime = runtime; + publisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime, + ...(runtime ? {} : { lifecycleReason: 'no_publisher_wallets' as const }), + }); } catch (err: any) { + publisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime: null, + lifecycleReason: 'publisher_startup_failed', + }); log(`Async publisher startup failed: ${err?.message ?? String(err)}`); } })(); @@ -3369,6 +3384,7 @@ export async function runDaemonInner( admissionStats, emitMemoryGraphChanged, emitNotification, + publisherAvailability, ); } catch (err: any) { // Single top-level error→HTTP mapping (in http-utils.ts diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index b525b44fa3..8b37447315 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -16,7 +16,7 @@ import type { OperationTracker, } from '@origintrail-official/dkg-node-ui'; import type { DkgConfig, loadNetworkConfig } from '../../config.js'; -import type { createPublisherControlFromStore, PublisherRuntime } from '../../publisher-runner.js'; +import type { AsyncPublisherAvailability, createPublisherControlFromStore, PublisherRuntime } from '../../publisher-runner.js'; import type { ExtractionStatusRecord } from '../../extraction-status.js'; import type { FileStore } from '../../file-store.js'; import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; @@ -60,6 +60,8 @@ export interface RequestContext { agent: DKGAgent; publisherControl: ReturnType; publisherRuntime: PublisherRuntime | null; + /** Lifecycle-owned publisher state; optional for direct route embeddings/tests. */ + publisherAvailability?: AsyncPublisherAvailability; config: DkgConfig; startedAt: number; dashDb: DashboardDB; diff --git a/packages/cli/src/daemon/routes/epcis.ts b/packages/cli/src/daemon/routes/epcis.ts index 14d8ca9b35..32bb1978a9 100644 --- a/packages/cli/src/daemon/routes/epcis.ts +++ b/packages/cli/src/daemon/routes/epcis.ts @@ -103,7 +103,7 @@ import { slotEntryPoint, CLI_NPM_PACKAGE, } from '../../config.js'; -import { createPublisherControlFromStore, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; +import { createPublisherControlFromStore, resolveAsyncPublisherAvailability, startPublisherRuntimeIfEnabled, type PublisherRuntime } from '../../publisher-runner.js'; import { createCatchupRunner, type CatchupJobResult, type CatchupRunner } from '../../catchup-runner.js'; import { loadTokens, httpAuthGuard, extractBearerToken } from '../../auth.js'; import { ExtractionPipelineRegistry } from '@origintrail-official/dkg-core'; @@ -490,16 +490,24 @@ export async function handleEpcisRoutes(ctx: RequestContext): Promise { // POST /api/epcis/capture { contextGraphId?, subGraphName?, epcisDocument, publishOptions? } if (req.method === "POST" && path === "/api/epcis/capture") { - if (!config.publisher?.enabled) { + const publisherAvailability = resolveAsyncPublisherAvailability({ + config, + runtime: publisherRuntime, + lifecycleAvailability: ctx.publisherAvailability, + }); + if (!publisherAvailability.available && publisherAvailability.reason === 'publisher_disabled') { return jsonResponse(res, 503, { error: "PublisherDisabled", message: "Async EPCIS capture requires publisher.enabled=true", }); } - if (!publisherRuntime || publisherRuntime.walletIds.length === 0) { + if (!publisherAvailability.available) { return jsonResponse(res, 503, { error: "PublisherUnavailable", message: "Async EPCIS capture requires the publisher runtime to be running with at least one configured publisher wallet", + reason: publisherAvailability.reason, + retryable: publisherAvailability.retryable, + operatorActionRequired: publisherAvailability.operatorActionRequired, }); } const body = await readBody(req); diff --git a/packages/cli/src/daemon/routes/knowledge-assets.ts b/packages/cli/src/daemon/routes/knowledge-assets.ts index 76cc9b379d..16bdc91a81 100644 --- a/packages/cli/src/daemon/routes/knowledge-assets.ts +++ b/packages/cli/src/daemon/routes/knowledge-assets.ts @@ -49,6 +49,7 @@ import { SMALL_BODY_BYTES, } from "../http-utils.js"; import { validatePreSignedAuthorAttestation } from "./memory.js"; +import { resolveAsyncPublisherAvailability } from "../../publisher-runner.js"; import { recordAssertionActivity, recordConvictionCostCovered } from "../activity-notification.js"; import { handleKaImportArtifactResolve, @@ -1347,6 +1348,20 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< // to 400 (parity with the legacy publish path). if (layer === "vm" && verb === "publish-async") { try { + const publisherAvailability = resolveAsyncPublisherAvailability({ + config: ctx.config, + runtime: ctx.publisherRuntime, + lifecycleAvailability: ctx.publisherAvailability, + }); + if (!publisherAvailability.available) { + return jsonResponse(res, 503, { + code: "async_publisher_unavailable", + error: "The asynchronous publisher cannot accept jobs on this node.", + reason: publisherAvailability.reason, + retryable: publisherAvailability.retryable, + operatorActionRequired: publisherAvailability.operatorActionRequired, + }); + } const opts = resolveStandaloneVmPublishOptions(ctx, parsed); if (opts === null) return; const publishOptions = opts; diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index a69d7c951a..adac992a45 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -24,6 +24,50 @@ export interface PublisherRuntimeWallet { readonly identityId: bigint; } +export type AsyncPublisherUnavailableReason = + | 'publisher_disabled' + | 'publisher_starting' + | 'no_publisher_wallets' + | 'publisher_startup_failed'; + +export type AsyncPublisherAvailability = + | { available: true } + | { + available: false; + reason: AsyncPublisherUnavailableReason; + retryable: boolean; + operatorActionRequired: boolean; + }; + +/** + * Canonical readiness boundary for every async-ingress route. Lifecycle may + * supply an explicit starting/failure state; otherwise the runtime/config + * shape is classified consistently for direct route tests and embedded users. + */ +export function resolveAsyncPublisherAvailability(args: { + config: DkgConfig; + runtime: PublisherRuntime | null; + lifecycleReason?: AsyncPublisherUnavailableReason; + lifecycleAvailability?: AsyncPublisherAvailability; +}): AsyncPublisherAvailability { + if (args.lifecycleAvailability) return args.lifecycleAvailability; + if (args.runtime?.walletIds.length) return { available: true }; + const reason = args.lifecycleReason + ?? (args.runtime + ? 'no_publisher_wallets' + : args.config.publisher?.enabled + ? 'publisher_startup_failed' + : 'publisher_disabled'); + return { + available: false, + reason, + // Only the in-progress state can recover from the same client retry without + // operator/config/daemon intervention. + retryable: reason === 'publisher_starting', + operatorActionRequired: reason !== 'publisher_starting', + }; +} + export interface PublisherInspector { readonly publisher: AsyncLiftPublisher; readonly stop: () => Promise; diff --git a/packages/cli/test/helpers/live-daemon.ts b/packages/cli/test/helpers/live-daemon.ts index b2ddf0846e..683b1ba5f9 100644 --- a/packages/cli/test/helpers/live-daemon.ts +++ b/packages/cli/test/helpers/live-daemon.ts @@ -53,6 +53,8 @@ function uniquePort(base: number): number { export interface StartDaemonOpts { authEnabled?: boolean; + /** Enable a real async publisher and seed its wallet file. */ + publisherEnabled?: boolean; /** Extra keys merged into config.json (e.g. preset contextGraphs). */ extraConfig?: Record; readyTimeoutMs?: number; @@ -89,6 +91,7 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise = { ...process.env, @@ -150,6 +160,30 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise l.trim()).find((l) => l.length > 0 && !l.startsWith('#')) ?? null; if (!daemon.token) throw new Error('auth enabled but no token written'); } + if (opts.publisherEnabled) { + // `/api/status` becomes ready before the zero-delay publisher startup task. + // Poll the real async-ingress gate so tests never race `publisher_starting`. + for (let i = 0; i < 60; i += 1) { + const res = await fetch( + `${daemon.base}/api/knowledge-assets/publisher-readiness/wm/probe/vm/publish-async`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(daemon.token ? { Authorization: `Bearer ${daemon.token}` } : {}), + }, + body: '{}', + }, + ); + const body = await res.json().catch(() => ({})) as { code?: string; reason?: string }; + if (body.code !== 'async_publisher_unavailable') break; + if (body.reason !== 'publisher_starting') { + throw new Error(`Async publisher failed readiness: ${body.reason ?? res.status}`); + } + await sleep(100); + if (i === 59) throw new Error('Async publisher did not become ready in time'); + } + } return daemon; } diff --git a/packages/cli/test/knowledge-assets-1116-share-errors.test.ts b/packages/cli/test/knowledge-assets-1116-share-errors.test.ts index 41965ec74a..55bf7e358d 100644 --- a/packages/cli/test/knowledge-assets-1116-share-errors.test.ts +++ b/packages/cli/test/knowledge-assets-1116-share-errors.test.ts @@ -62,6 +62,12 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { agentOverrides: Record = {}, routeOverrides: { requestToken?: string; requestAgentAddress?: string } = {}, publisherControl: Record = {}, + publisherRuntime: unknown = { + walletIds: ['0x1111111111111111111111111111111111111111'], + wallets: [{ address: '0x1111111111111111111111111111111111111111' }], + }, + config: Record = {}, + publisherAvailability?: unknown, ) { const agent = { async listContextGraphs() { @@ -90,8 +96,9 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { res, agent, publisherControl, - publisherRuntime: null, - config: {}, + publisherRuntime, + config, + publisherAvailability, startedAt: Date.now(), dashDb: { insertNotification: () => 1 }, opWallets: {}, @@ -382,6 +389,73 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { expect(enqueueCalls).toBe(0); }); + it('vm/publish-async rejects before persisting when no runtime can claim jobs', async () => { + let resolved = 0; + let enqueued = 0; + await startWith({}, { + resolveFinalizedAssertionVmPublishIntent: async () => { resolved += 1; }, + }, {}, { + enqueueKnowledgeAssetVmPublish: async () => { enqueued += 1; }, + }, null); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + code: 'async_publisher_unavailable', + reason: 'publisher_disabled', + retryable: false, + operatorActionRequired: true, + }); + expect(resolved).toBe(0); + expect(enqueued).toBe(0); + }); + + it('vm/publish-async treats an empty-wallet runtime as operator-actionable', async () => { + let enqueued = 0; + await startWith({}, {}, {}, { + enqueueKnowledgeAssetVmPublish: async () => { enqueued += 1; }, + }, { walletIds: [], wallets: [] }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + code: 'async_publisher_unavailable', + reason: 'no_publisher_wallets', + retryable: false, + operatorActionRequired: true, + }); + expect(enqueued).toBe(0); + }); + + it('vm/publish-async uses lifecycle no-wallet state when the runtime is null', async () => { + let enqueued = 0; + await startWith({}, {}, {}, { + enqueueKnowledgeAssetVmPublish: async () => { enqueued += 1; }, + }, null, { publisher: { enabled: true } }, { + available: false, + reason: 'no_publisher_wallets', + retryable: false, + operatorActionRequired: true, + }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + reason: 'no_publisher_wallets', retryable: false, operatorActionRequired: true, + }); + expect(enqueued).toBe(0); + }); + + it('vm/publish-async classifies an unknown startup failure as operator-actionable', async () => { + await startWith({}, {}, {}, {}, null, { publisher: { enabled: true } }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(503); + expect(res.body).toMatchObject({ + reason: 'publisher_startup_failed', retryable: false, operatorActionRequired: true, + }); + }); + it('vm/publish-async rejects a missing real share snapshot before enqueue', async () => { const store = await createTripleStore({ backend: 'oxigraph' }); let enqueueCalls = 0; diff --git a/packages/cli/test/publisher-availability.test.ts b/packages/cli/test/publisher-availability.test.ts new file mode 100644 index 0000000000..6c549edb7d --- /dev/null +++ b/packages/cli/test/publisher-availability.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { resolveAsyncPublisherAvailability, type PublisherRuntime } from '../src/publisher-runner.js'; + +const runtime = (wallets: unknown[]): PublisherRuntime => ({ + wallets, + walletIds: wallets.map((_, index) => String(index)), +} as unknown as PublisherRuntime); + +describe('resolveAsyncPublisherAvailability', () => { + it('classifies permanent configuration states as operator-actionable', () => { + expect(resolveAsyncPublisherAvailability({ config: {}, runtime: null })).toMatchObject({ + available: false, reason: 'publisher_disabled', retryable: false, operatorActionRequired: true, + }); + expect(resolveAsyncPublisherAvailability({ + config: { publisher: { enabled: true } } as any, + runtime: runtime([]), + })).toMatchObject({ + available: false, reason: 'no_publisher_wallets', retryable: false, operatorActionRequired: true, + }); + expect(resolveAsyncPublisherAvailability({ + config: { publisher: { enabled: true } } as any, + runtime: null, + lifecycleReason: 'publisher_startup_failed', + })).toMatchObject({ + available: false, reason: 'publisher_startup_failed', retryable: false, operatorActionRequired: true, + }); + }); + + it('marks only an in-progress startup as retryable and a funded runtime as ready', () => { + expect(resolveAsyncPublisherAvailability({ + config: { publisher: { enabled: true } } as any, + runtime: null, + lifecycleReason: 'publisher_starting', + })).toMatchObject({ + available: false, reason: 'publisher_starting', retryable: true, operatorActionRequired: false, + }); + expect(resolveAsyncPublisherAvailability({ config: {}, runtime: runtime([{}]) })).toEqual({ available: true }); + }); +}); diff --git a/packages/cli/test/source-worker-daemon-client.test.ts b/packages/cli/test/source-worker-daemon-client.test.ts index 16136bee5c..11e1776d44 100644 --- a/packages/cli/test/source-worker-daemon-client.test.ts +++ b/packages/cli/test/source-worker-daemon-client.test.ts @@ -17,7 +17,7 @@ describe('source worker daemon client (real daemon)', () => { let daemon: LiveDaemon; beforeAll(async () => { - daemon = await startLiveDaemon(); + daemon = await startLiveDaemon({ publisherEnabled: true }); const created = await postJson(daemon, '/api/context-graph/create', { id: CG, name: CG, accessPolicy: 0 }); expect(created.status, `CG create failed: ${JSON.stringify(created.body)}`).toBeLessThan(300); }, 120_000); diff --git a/packages/cli/test/source-worker-runner.test.ts b/packages/cli/test/source-worker-runner.test.ts index 827fc72d33..7b22834a88 100644 --- a/packages/cli/test/source-worker-runner.test.ts +++ b/packages/cli/test/source-worker-runner.test.ts @@ -27,7 +27,7 @@ describe('source worker runner (real daemon)', () => { beforeAll(async () => { console.log = () => undefined; - daemon = await startLiveDaemon(); + daemon = await startLiveDaemon({ publisherEnabled: true }); const created = await postJson(daemon, '/api/context-graph/create', { id: CG, name: CG, accessPolicy: 0 }); expect(created.status, `CG create failed: ${JSON.stringify(created.body)}`).toBeLessThan(300); const sg = await postJson(daemon, '/api/sub-graph/create', { contextGraphId: CG, subGraphName: 'sg-1' }); diff --git a/scripts/devnet-test-issue-1576-publisher-readiness.sh b/scripts/devnet-test-issue-1576-publisher-readiness.sh new file mode 100755 index 0000000000..5431b4e6e4 --- /dev/null +++ b/scripts/devnet-test-issue-1576-publisher-readiness.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1576. A temporary edge node is started with +# publisher.enabled=true but no publisher wallet. publish-async must return 503 +# before persistence, and the durable publisher job count must not change. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +NODE="${PUBLISHER_READINESS_NODE:-7}" +NODE_DIR="$DEVNET_DIR/node$NODE" +CG="${DEVNET_CONTEXT_GRAPH:-devnet-test}" + +fail() { echo "[#1576] FAIL: $*" >&2; exit 1; } +cleanup() { + "$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null 2>&1 || true + rm -rf "$NODE_DIR" +} +trap cleanup EXIT INT TERM + +[[ -d "$DEVNET_DIR/node1" ]] || fail "start the baseline devnet first" +[[ ! -d "$NODE_DIR" ]] || fail "$NODE_DIR already exists; choose PUBLISHER_READINESS_NODE" +"$ROOT/scripts/devnet.sh" addnode "$NODE" edge >/dev/null +"$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null + +NODE_DIR="$NODE_DIR" node --input-type=module <<'NODE' +import { readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +const path = join(process.env.NODE_DIR, 'config.json'); +const config = JSON.parse(readFileSync(path, 'utf8')); +config.publisher = { ...(config.publisher ?? {}), enabled: true }; +writeFileSync(path, JSON.stringify(config, null, 2)); +rmSync(join(process.env.NODE_DIR, 'publisher-wallets.json'), { force: true }); +NODE +"$ROOT/scripts/devnet.sh" restart-node "$NODE" >/dev/null + +. "$ROOT/scripts/devnet-lib.sh" +for _ in $(seq 1 90); do + [[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] && break + sleep 1 +done +[[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] || fail "temporary node did not become ready" +api "$NODE" POST /api/identity/ensure '{}' >/dev/null || true + +jobs_before_body="$(body_of "$(api "$NODE" GET /api/publisher/jobs)")" +jobs_before="$(JOBS="$jobs_before_body" node -e 'const j=JSON.parse(process.env.JOBS);process.stdout.write(String((j.jobs||[]).length))')" + +name="issue-1576-$(date +%s)-$$" +subject="urn:issue:1576:$name" +api "$NODE" POST /api/knowledge-assets "{\"contextGraphId\":\"$CG\",\"name\":\"$name\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/write" \ + "{\"contextGraphId\":\"$CG\",\"quads\":[{\"subject\":\"$subject\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"publisher readiness probe\\\"\",\"graph\":\"\"}]}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/finalize" "{\"contextGraphId\":\"$CG\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/swm/share" "{\"contextGraphId\":\"$CG\"}" >/dev/null + +response="$(api "$NODE" POST "/api/knowledge-assets/$name/vm/publish-async" "{\"contextGraphId\":\"$CG\"}")" +[[ "$(code_of "$response")" == 503 ]] || fail "expected 503, got $(code_of "$response"): $(body_of "$response")" +body="$(body_of "$response")" +[[ "$(field "$body" code)" == async_publisher_unavailable ]] || fail "wrong stable error code: $body" +[[ "$(field "$body" reason)" == no_publisher_wallets ]] || fail "wrong unavailable reason: $body" +[[ "$(field "$body" retryable)" == false ]] || fail "no-wallet state was advertised retryable" + +jobs_after_body="$(body_of "$(api "$NODE" GET /api/publisher/jobs)")" +jobs_after="$(JOBS="$jobs_after_body" node -e 'const j=JSON.parse(process.env.JOBS);process.stdout.write(String((j.jobs||[]).length))')" +[[ "$jobs_after" == "$jobs_before" ]] || fail "job count grew from $jobs_before to $jobs_after" +echo "[#1576] PASS: unavailable publisher rejected before durable enqueue" From f89226e2ba467b7582df279f8bfeb21fd04cab69 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 08:59:35 +0200 Subject: [PATCH 07/12] fix(chain): retry reads after full-pool throttling (#1561) (#1628) * fix(chain): retry reads after full-pool throttling * fix(chain): scope throttle retries to critical reads * fix(chain): retry only all-throttled receipt block reads * refactor(chain): contain full-pool throttle retry policy --------- Co-authored-by: Branimir Rakic --- .../chain/src/chain-rpc-transport-error.ts | 1 - packages/chain/src/evm-adapter-base.ts | 10 +- packages/chain/src/evm-adapter-rpc.ts | 8 ++ packages/chain/src/rpc-failover-client.ts | 50 +++++++++- ...vm-adapter-tip-read-carveouts.unit.test.ts | 16 ++++ .../test/readwithfailover-loop.unit.test.ts | 45 ++++++++- ...vnet-test-issue-1561-rpc-throttle-retry.sh | 94 +++++++++++++++++++ 7 files changed, 216 insertions(+), 8 deletions(-) create mode 100755 scripts/devnet-test-issue-1561-rpc-throttle-retry.sh diff --git a/packages/chain/src/chain-rpc-transport-error.ts b/packages/chain/src/chain-rpc-transport-error.ts index c815ea5de1..1b0039ab17 100644 --- a/packages/chain/src/chain-rpc-transport-error.ts +++ b/packages/chain/src/chain-rpc-transport-error.ts @@ -61,7 +61,6 @@ export class ChainRpcTransportError extends Error { readonly rpcUrls?: readonly string[]; readonly txHash?: string; - constructor( code: ChainRpcTransportCode, message: string, diff --git a/packages/chain/src/evm-adapter-base.ts b/packages/chain/src/evm-adapter-base.ts index 0b064cf3c9..5162ac9443 100644 --- a/packages/chain/src/evm-adapter-base.ts +++ b/packages/chain/src/evm-adapter-base.ts @@ -2591,7 +2591,15 @@ export class EVMChainAdapterBase { // endpoint that has it; best-effort `0` only when EVERY endpoint lacks it and // no transport error occurred. Non-retryable / transport-exhaustion errors // propagate (never masked as a bogus `0`). Order-independent — see the helper. - const block = await this.readProviderRetryingNull('getBlock', (p) => p.getBlock(blockNumber)); + const block = await this.rpcFailover.read( + 'getBlock', + (p) => p.getBlock(blockNumber), + { + rpcUsageConsumer: 'getBlock', + isEmptyResult: (value) => value == null, + endpointSetRetry: 'all-throttled', + }, + ); return block?.timestamp != null ? Number(block.timestamp) : 0; } diff --git a/packages/chain/src/evm-adapter-rpc.ts b/packages/chain/src/evm-adapter-rpc.ts index 896143a852..277852f847 100644 --- a/packages/chain/src/evm-adapter-rpc.ts +++ b/packages/chain/src/evm-adapter-rpc.ts @@ -153,6 +153,14 @@ export function isRetryableRpcError(err: unknown): boolean { .test(msg); } +/** Canonical classifier for provider throttling, shared by failover policy. */ +export function isThrottleRpcError(err: unknown): boolean { + if (err instanceof Error) enrichEvmError(err); + const status = errorStatus(err); + const message = errorMessage(err).toLowerCase(); + return status === 429 || /\b429\b|too many requests|rate[ -]?limit|throttl/.test(message); +} + export function assertSuccessfulReceipt(receipt: ethers.TransactionReceipt, label: string): void { if (receipt.status !== 0) return; const err = new Error(`${label} tx ${receipt.hash} was mined but reverted (status=0)`); diff --git a/packages/chain/src/rpc-failover-client.ts b/packages/chain/src/rpc-failover-client.ts index c0297b6cf7..a74b6d0587 100644 --- a/packages/chain/src/rpc-failover-client.ts +++ b/packages/chain/src/rpc-failover-client.ts @@ -41,7 +41,7 @@ import { JsonRpcProvider, Wallet, Contract, ethers } from 'ethers'; import { withSpan, getMetrics } from '@origintrail-official/dkg-core'; -import { withTimeout, isRetryableRpcError, isKnownTransactionError } from './evm-adapter-rpc.js'; +import { withTimeout, isRetryableRpcError, isThrottleRpcError, isKnownTransactionError, sleep } from './evm-adapter-rpc.js'; import { errorCode, errorMessage } from './evm-adapter-errors.js'; import { noteRpcFailover, noteRpcExhaustion, notePreferredEndpoint, noteRpcServed, rpcHost } from './rpc-failover-log.js'; import { EndpointStickiness, type StickinessIntent } from './endpoint-stickiness.js'; @@ -117,6 +117,21 @@ export interface ReadOpts { * sentinel polluting stickiness or telemetry. */ isEmptyResult?: (value: unknown) => boolean; + /** Retry a complete endpoint pass only when every failure was a throttle. */ + endpointSetRetry?: 'all-throttled'; +} + +type ProviderSetExhaustionKind = 'all-throttled' | 'mixed'; + +/** Internal exhaustion detail used only while deciding whether to retry a pass. */ +class ProviderSetExhaustedError extends ChainRpcTransportError { + constructor( + message: string, + readonly exhaustionKind: ProviderSetExhaustionKind, + opts: { cause: unknown; rpcUrls: readonly string[] }, + ) { + super('RPC_ENDPOINTS_EXHAUSTED', message, opts); + } } /** @@ -142,6 +157,9 @@ export interface RpcFailoverClientOptions { validateEndpoint?: ValidateEndpointFn; /** Endpoint-stickiness configuration (Mechanism B). */ stickiness?: StickinessOptions; + /** Full-pool retries after every endpoint reports a transient throttle. */ + readThrottleRetries?: number; + readThrottleBackoffMs?: number; } export interface StickinessOptions { @@ -205,6 +223,8 @@ export class RpcFailoverClient { private readonly stickiness: EndpointStickiness; /** Optional per-endpoint transport preflight (from `options.validateEndpoint`). */ private readonly validateEndpoint?: ValidateEndpointFn; + private readonly readThrottleRetries: number; + private readonly readThrottleBackoffMs: number; constructor( private readonly getEndpoints: () => RpcEndpoint[], @@ -219,6 +239,8 @@ export class RpcFailoverClient { options?: RpcFailoverClientOptions, ) { this.validateEndpoint = options?.validateEndpoint; + this.readThrottleRetries = options?.readThrottleRetries ?? 2; + this.readThrottleBackoffMs = options?.readThrottleBackoffMs ?? 250; const stickiness = options?.stickiness; const isEnabled = stickiness?.isEnabled ?? (stickiness?.enabled !== undefined ? () => stickiness.enabled as boolean : () => true); @@ -288,7 +310,7 @@ export class RpcFailoverClient { fn: (provider: JsonRpcProvider) => Promise, opts?: ReadOpts, ): Promise { - const run = () => this.runAcrossProviders( + const runPass = () => this.runAcrossProviders( label, fn, opts?.isRetryable ?? isRetryableRpcError, @@ -296,6 +318,9 @@ export class RpcFailoverClient { opts?.skipPreferred ?? false, opts?.isEmptyResult, ); + const run = opts?.endpointSetRetry === 'all-throttled' + ? () => this.withThrottleRetries(runPass) + : runPass; return opts?.rpcUsageConsumer ? withRpcUsageConsumer(opts.rpcUsageConsumer, run) : run(); } @@ -666,6 +691,7 @@ export class RpcFailoverClient { const attempts = this.stickiness.attempts(canonical, intent); const capMs = resolveCapMs(policy, canonical.length); let lastRetryable: unknown; + let allEndpointsThrottled = true; let sawEmpty = false; let lastEmpty: T | undefined; for (let i = 0; i < attempts.length; i += 1) { @@ -691,6 +717,7 @@ export class RpcFailoverClient { // telemetry. Try the next endpoint; if EVERY endpoint is empty (and none // errored) the empty value itself is the honest answer. sawEmpty = true; + allEndpointsThrottled = false; lastEmpty = out; continue; } @@ -700,6 +727,7 @@ export class RpcFailoverClient { } catch (err) { if (!isRetryable(err)) throw err; lastRetryable = err; + if (!isThrottleRpcError(err)) allEndpointsThrottled = false; attempt.recordFailure(); // de-prefer a failed backend if (!isLast) { noteRpcFailover(label, endpoint.rpcUrl, err, attempts[i + 1].endpoint.rpcUrl); @@ -725,7 +753,7 @@ export class RpcFailoverClient { ? errorMessage(lastRetryable) : `${label} read failed on all configured RPC endpoints ` + `(${canonical.map((e) => rpcHost(e.rpcUrl)).join(', ')}): ${errorMessage(lastRetryable)}`; - throw new ChainRpcTransportError('RPC_ENDPOINTS_EXHAUSTED', message, { + throw new ProviderSetExhaustedError(message, allEndpointsThrottled ? 'all-throttled' : 'mixed', { cause: lastRetryable, rpcUrls: canonical.map((e) => e.rpcUrl), }); @@ -743,6 +771,22 @@ export class RpcFailoverClient { ); } + private async withThrottleRetries(run: () => Promise): Promise { + for (let retry = 0; ; retry += 1) { + try { + return await run(); + } catch (error) { + // A raw 429 from a caller-supplied non-retryable classifier must retain + // its no-failover/no-retry contract. Only retry a typed FULL-POOL + // exhaustion produced by runAcrossProviders. + if (!(error instanceof ProviderSetExhaustedError) + || error.exhaustionKind !== 'all-throttled' + || retry >= this.readThrottleRetries) throw error; + await sleep(this.readThrottleBackoffMs * (2 ** retry)); + } + } + } + /** * Rebind a CONTRACT to `runner` (a provider for a view read) for one * per-endpoint attempt, leaving the caller's boot-bound handle untouched. The diff --git a/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts b/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts index 328a4a5c0e..5281aca031 100644 --- a/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts +++ b/packages/chain/test/evm-adapter-tip-read-carveouts.unit.test.ts @@ -181,6 +181,22 @@ describe('getBlockTimestamp: a null (unimported) receipt block fails over instea await expect(a.getBlockTimestamp(123n)).rejects.toMatchObject({ code: 'RPC_ENDPOINTS_EXHAUSTED' }); }); + it('retries an all-429 receipt-block pass and returns the recovered timestamp', async () => { + const retryable429 = () => { const e: any = new Error('429 too many requests'); e.status = 429; return e; }; + const primary = recorder(async () => { throw retryable429(); }); + let backupAttempt = 0; + const backup = recorder(async () => { + backupAttempt += 1; + if (backupAttempt === 1) throw retryable429(); + return { timestamp: 42 }; + }); + const a = makeTwoEndpointAdapter({ getBlock: primary }, { getBlock: backup }); + + await expect(a.getBlockTimestamp(123n)).resolves.toBe(42); + expect(primary.calls).toHaveLength(2); + expect(backup.calls).toHaveLength(2); + }); + it('MIXED null + transport error PROPAGATES regardless of endpoint order (order-independent, round-6 🔴)', async () => { const retryable429 = () => { const e: any = new Error('429 too many requests'); e.status = 429; return e; }; // Order A: primary transport error, backup null. A transport failure occurred, diff --git a/packages/chain/test/readwithfailover-loop.unit.test.ts b/packages/chain/test/readwithfailover-loop.unit.test.ts index 569b674d03..17a504e1f5 100644 --- a/packages/chain/test/readwithfailover-loop.unit.test.ts +++ b/packages/chain/test/readwithfailover-loop.unit.test.ts @@ -28,7 +28,7 @@ */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { EVMChainAdapter, type EVMAdapterConfig } from '../src/evm-adapter.js'; -import { RpcFailoverClient, type SignPopulatedFn } from '../src/rpc-failover-client.js'; +import { RpcFailoverClient, type RpcFailoverClientOptions, type SignPopulatedFn } from '../src/rpc-failover-client.js'; import { isChainRpcTransportError } from '../src/chain-rpc-transport-error.js'; import { getRpcFailoverStats, _resetRpcFailoverStatsForTest } from '../src/rpc-failover-log.js'; import { RPC_READ_STALL_TIMEOUT_MS } from '../src/evm-adapter-constants.js'; @@ -79,11 +79,12 @@ const NEVER_SIGN: SignPopulatedFn = async () => { * is the exact shape the adapter constructs it with, minus the adapter — so a * read failover regression is caught without a god-object back-reference. */ -function makeClient(providers: unknown[], rpcUrls: string[], signPopulated: SignPopulatedFn = NEVER_SIGN): RpcFailoverClient { +function makeClient(providers: unknown[], rpcUrls: string[], signPopulated: SignPopulatedFn = NEVER_SIGN, options?: RpcFailoverClientOptions): RpcFailoverClient { return new RpcFailoverClient( () => providers.map((p, i) => ({ provider: p as any, rpcUrl: rpcUrls[i] })), signPopulated, () => 'evm:31337', + options, ); } @@ -104,7 +105,9 @@ describe('RpcFailoverClient.read — read-failover loop logic (bare-mock, #1336) it('exhausts ALL endpoints → ChainRpcTransportError RPC_ENDPOINTS_EXHAUSTED, one attempt each', async () => { const primary = { read: recorder(async () => { throw retryable429(); }) }; const backup = { read: recorder(async () => { throw retryable429(); }) }; - const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example']); + const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example'], NEVER_SIGN, { + readThrottleRetries: 0, + }); let thrown: any; try { await client.read('unit read', (p: any) => p.read()); } catch (e) { thrown = e; } @@ -117,6 +120,42 @@ describe('RpcFailoverClient.read — read-failover loop logic (bare-mock, #1336) expect(backup.read.calls).toHaveLength(1); }); + it('backs off and retries the full pool when every endpoint returns 429', async () => { + let round = 0; + const primary = { read: recorder(async () => { throw retryable429(); }) }; + const backup = { read: recorder(async () => { + round += 1; + if (round === 1) throw retryable429(); + return 'recovered'; + }) }; + const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example'], NEVER_SIGN, { + readThrottleRetries: 2, + readThrottleBackoffMs: 1, + }); + + await expect(client.read('getBlock', (provider: any) => provider.read(), { + endpointSetRetry: 'all-throttled', + })) + .resolves.toBe('recovered'); + expect(primary.read.calls).toHaveLength(2); + expect(backup.read.calls).toHaveLength(2); + }); + + it('does not retry a mixed timeout plus 429 endpoint exhaustion', async () => { + const primary = { read: recorder(async () => { const error: any = new Error('timed out'); error.code = 'TIMEOUT'; throw error; }) }; + const backup = { read: recorder(async () => { throw retryable429(); }) }; + const client = makeClient([primary, backup], ['https://primary.example', 'https://backup.example'], NEVER_SIGN, { + readThrottleRetries: 2, + readThrottleBackoffMs: 1, + }); + + await expect(client.read('getBlock', (provider: any) => provider.read(), { + endpointSetRetry: 'all-throttled', + })).rejects.toMatchObject({ code: 'RPC_ENDPOINTS_EXHAUSTED' }); + expect(primary.read.calls).toHaveLength(1); + expect(backup.read.calls).toHaveLength(1); + }); + it('single-RPC: a retryable failure still stamps RPC_ENDPOINTS_EXHAUSTED but keeps the original message verbatim', async () => { const only = { read: recorder(async () => { throw new Error('connect ECONNREFUSED 127.0.0.1:8545'); }) }; const client = makeClient([only], ['https://only.example']); diff --git a/scripts/devnet-test-issue-1561-rpc-throttle-retry.sh b/scripts/devnet-test-issue-1561-rpc-throttle-retry.sh new file mode 100755 index 0000000000..ff4f5f58fb --- /dev/null +++ b/scripts/devnet-test-issue-1561-rpc-throttle-retry.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1561. A temporary node uses two logical RPC +# endpoints behind a fault proxy. The first eth_getBlockByNumber on each endpoint +# returns 429 (one complete throttled pool); the next pool pass forwards and the +# publish must confirm. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +UPSTREAM="${DEVNET_RPC:-http://127.0.0.1:8545}" +PROXY_PORT="${THROTTLE_PROXY_PORT:-18561}" +NODE="${THROTTLE_TEST_NODE:-7}" +NODE_DIR="$DEVNET_DIR/node$NODE" +CG="${DEVNET_CONTEXT_GRAPH:-devnet-test}" +proxy_pid='' + +fail() { echo "[#1561] FAIL: $*" >&2; exit 1; } +cleanup() { + "$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null 2>&1 || true + rm -rf "$NODE_DIR" + [[ -n "$proxy_pid" ]] && kill "$proxy_pid" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM +[[ ! -d "$NODE_DIR" ]] || fail "$NODE_DIR already exists" + +UPSTREAM="$UPSTREAM" PROXY_PORT="$PROXY_PORT" node --input-type=module <<'NODE' & +import http from 'node:http'; +const counts = new Map(); +http.createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/stats') { + res.setHeader('content-type', 'application/json'); + return res.end(JSON.stringify(Object.fromEntries(counts))); + } + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = Buffer.concat(chunks); + let method = ''; + try { method = JSON.parse(body.toString()).method; } catch {} + const key = `${req.url}:${method}`; + const count = counts.get(key) ?? 0; + counts.set(key, count + 1); + if (method === 'eth_getBlockByNumber' && count === 0) { + res.statusCode = 429; + res.setHeader('content-type', 'application/json'); + return res.end(JSON.stringify({ error: 'Too Many Requests' })); + } + const upstream = await fetch(process.env.UPSTREAM, { + method: 'POST', headers: { 'content-type': 'application/json' }, body, + }); + res.statusCode = upstream.status; + res.end(Buffer.from(await upstream.arrayBuffer())); +}).listen(Number(process.env.PROXY_PORT), '127.0.0.1'); +NODE +proxy_pid=$! +sleep 1 +curl -fsS "http://127.0.0.1:$PROXY_PORT/stats" >/dev/null || fail "fault proxy did not start" + +"$ROOT/scripts/devnet.sh" addnode "$NODE" edge >/dev/null +"$ROOT/scripts/devnet.sh" stop-node "$NODE" >/dev/null +NODE_DIR="$NODE_DIR" PROXY_PORT="$PROXY_PORT" node --input-type=module <<'NODE' +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +const path = join(process.env.NODE_DIR, 'config.json'); +const config = JSON.parse(readFileSync(path, 'utf8')); +const base = `http://127.0.0.1:${process.env.PROXY_PORT}`; +config.chain.rpcUrl = `${base}/a`; +config.chain.rpcUrls = [`${base}/a`, `${base}/b`]; +writeFileSync(path, JSON.stringify(config, null, 2)); +NODE +"$ROOT/scripts/devnet.sh" restart-node "$NODE" >/dev/null + +. "$ROOT/scripts/devnet-lib.sh" +for _ in $(seq 1 90); do + [[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] && break + sleep 1 +done +[[ "$(code_of "$(api "$NODE" GET /api/status)")" == 200 ]] || fail "temporary node not ready" +api "$NODE" POST /api/identity/ensure '{}' >/dev/null || true + +name="issue-1561-$(date +%s)-$$"; subject="urn:issue:1561:$name" +api "$NODE" POST /api/knowledge-assets "{\"contextGraphId\":\"$CG\",\"name\":\"$name\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/write" \ + "{\"contextGraphId\":\"$CG\",\"quads\":[{\"subject\":\"$subject\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"429 recovery probe\\\"\",\"graph\":\"\"}]}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/wm/finalize" "{\"contextGraphId\":\"$CG\"}" >/dev/null +api "$NODE" POST "/api/knowledge-assets/$name/swm/share" "{\"contextGraphId\":\"$CG\"}" >/dev/null +result="$(api "$NODE" POST "/api/knowledge-assets/$name/vm/publish" "{\"contextGraphId\":\"$CG\"}")" +[[ "$(code_of "$result")" == 200 ]] || fail "publish failed: $(body_of "$result")" +[[ "$(field "$(body_of "$result")" status)" == confirmed ]] || fail "publish not confirmed" + +stats="$(curl -fsS "http://127.0.0.1:$PROXY_PORT/stats")" +STATS="$stats" node -e 'const s=JSON.parse(process.env.STATS); for (const p of ["/a","/b"]) { const n=s[`${p}:eth_getBlockByNumber`]||0; if(n<2) throw new Error(`${p} getBlock calls=${n}, expected throttle plus recovery`); }' \ + || fail "proxy did not observe a full throttled pass plus recovery: $stats" +echo "[#1561] PASS: publish recovered after both RPC endpoints returned 429" From 2222ed33147515f7c27e89f315c2ef107fb57297 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 09:00:01 +0200 Subject: [PATCH 08/12] fix(cli): verify and roll back torn global updates (#1567) (#1630) * fix(cli): verify and roll back torn global updates * fix(update): verify exact CLI version before commit --------- Co-authored-by: Branimir Rakic --- packages/cli/src/daemon/auto-update.ts | 68 +++++++++++++++---- packages/cli/test/rfc-41-bundle-b.test.ts | 57 +++++++++++++++- ...net-test-issue-1567-global-cli-rollback.sh | 43 ++++++++++++ 3 files changed, 153 insertions(+), 15 deletions(-) create mode 100755 scripts/devnet-test-issue-1567-global-cli-rollback.sh diff --git a/packages/cli/src/daemon/auto-update.ts b/packages/cli/src/daemon/auto-update.ts index 433138563c..194574161a 100644 --- a/packages/cli/src/daemon/auto-update.ts +++ b/packages/cli/src/daemon/auto-update.ts @@ -1543,11 +1543,9 @@ export async function performNpmUpdate( * version to `~/.dkg/previous-version` so `dkg rollback` (Edge * branch) has a target to reinstall. * - * Tradeoffs accepted (per RFC §7.2): - * - Non-atomic: a mid-install crash leaves the global state - * half-updated. Recovery is `npm install -g` re-run. - * - Network-dependent rollback: requires the npm registry to - * have the previous version available. + * The npm mutation itself is not atomic, so completion is followed by an + * executable/version self-check. A failed check immediately reinstalls the + * recorded previous version before the daemon is allowed to restart. * * The function returns `'updated'` after the npm install completes; * the caller is responsible for stopping the running daemon so the @@ -1617,14 +1615,7 @@ async function _performNpmUpdateInnerEdge( log(`Auto-update (npm-edge): running '${installCmd}'…`); try { const installStart = Date.now(); - await execAsyncIo(installCmd, { - encoding: "utf-8", - timeout: 300_000, - // Allow npm's progress / warning output to surface in the daemon - // log — operators tailing the log get real-time feedback on slow - // installs. stderr → stdout merge mirrors how npm itself runs - // interactively. - }); + await installGlobalCliVersion(execAsyncIo, targetVersion); const installMs = Date.now() - installStart; log(`Auto-update (npm-edge): npm install completed in ${installMs}ms.`); } catch (installErr: any) { @@ -1640,6 +1631,27 @@ async function _performNpmUpdateInnerEdge( return "failed"; } + try { + const reported = await verifyGlobalDkgVersion(execAsyncIo, targetVersion, 'self-check'); + log(`Auto-update (npm-edge): self-check passed (${reported}).`); + } catch (verifyErr: any) { + log(`Auto-update (npm-edge): post-install self-check failed — ${verifyErr?.message ?? verifyErr}.`); + if (!currentVersion) { + log('Auto-update (npm-edge): previous version is unknown; automatic rollback is unavailable.'); + return 'failed'; + } + const rollbackCmd = `npm install -g ${CLI_NPM_PACKAGE}@${currentVersion}`; + log(`Auto-update (npm-edge): rolling back with '${rollbackCmd}'…`); + try { + await installGlobalCliVersion(execAsyncIo, currentVersion); + const reported = await verifyGlobalDkgVersion(execAsyncIo, currentVersion, 'rollback'); + log(`Auto-update (npm-edge): rollback restored ${reported}.`); + } catch (rollbackErr: any) { + log(`Auto-update (npm-edge): CRITICAL rollback failed — ${rollbackErr?.message ?? rollbackErr}.`); + } + return 'failed'; + } + log( `Auto-update (npm-edge): ${CLI_NPM_PACKAGE}@${targetVersion} installed. ` + "Stop the daemon to restart from the new entry point.", @@ -1647,6 +1659,36 @@ async function _performNpmUpdateInnerEdge( return "updated"; } +type EdgeExec = typeof _autoUpdateIo.exec; + +async function installGlobalCliVersion(execIo: EdgeExec, version: string): Promise { + await execIo(`npm install -g ${CLI_NPM_PACKAGE}@${version}`, { + encoding: 'utf-8', + timeout: 300_000, + }); +} + +function parseReportedDkgVersion(output: string): string | undefined { + return output.match(/(?:^|\s)v?(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)(?=\s|$)/)?.[1]; +} + +async function verifyGlobalDkgVersion( + execIo: EdgeExec, + expectedVersion: string, + context: 'self-check' | 'rollback', +): Promise { + const { stdout, stderr } = await execIo('dkg --version', { + encoding: 'utf-8', + timeout: 30_000, + }); + const reported = `${stdout ?? ''} ${stderr ?? ''}`.trim(); + const parsed = parseReportedDkgVersion(reported); + if (parsed !== expectedVersion) { + throw new Error(`${context} expected ${expectedVersion}, got ${parsed ?? (reported || 'empty version output')}`); + } + return reported; +} + export async function checkForUpdate( au: ResolvedAutoUpdateConfig, log: (msg: string) => void, diff --git a/packages/cli/test/rfc-41-bundle-b.test.ts b/packages/cli/test/rfc-41-bundle-b.test.ts index 1e5df4ded1..e888b67a85 100644 --- a/packages/cli/test/rfc-41-bundle-b.test.ts +++ b/packages/cli/test/rfc-41-bundle-b.test.ts @@ -213,7 +213,10 @@ describe('performNpmUpdateEdge (Bundle B1b)', () => { execCalls = []; _autoUpdateIo.exec = ((cmd: string, opts?: any): Promise<{ stdout: string; stderr: string }> => { execCalls.push({ cmd, opts }); - return Promise.resolve({ stdout: '', stderr: '' }); + return Promise.resolve({ + stdout: cmd === 'dkg --version' ? 'dkg 10.0.0-rc.12' : '', + stderr: '', + }); }) as any; }); @@ -227,8 +230,9 @@ describe('performNpmUpdateEdge (Bundle B1b)', () => { expect(result).toBe('updated'); expect(readFileSync(join(dkgHome, 'previous-version'), 'utf-8')).toBe('10.0.0-rc.11'); - expect(execCalls).toHaveLength(1); + expect(execCalls).toHaveLength(2); expect(execCalls[0].cmd).toBe('npm install -g @origintrail-official/dkg@10.0.0-rc.12'); + expect(execCalls[1].cmd).toBe('dkg --version'); expect(log.calls.some((m) => m.includes('10.0.0-rc.11 → ~/.dkg/previous-version'))).toBe(true); expect(log.calls.some((m) => m.includes('install completed'))).toBe(true); }); @@ -261,6 +265,55 @@ describe('performNpmUpdateEdge (Bundle B1b)', () => { expect(readFileSync(join(dkgHome, 'previous-version'), 'utf-8')).toBe('10.0.0-rc.11'); }); + it('rolls back when the installed CLI cannot pass its self-check', async () => { + let versionChecks = 0; + _autoUpdateIo.exec = (async (cmd: string, opts?: any) => { + execCalls.push({ cmd, opts }); + if (cmd === 'dkg --version') { + versionChecks += 1; + if (versionChecks === 1) throw new Error('dkg: command not found'); + return { stdout: 'dkg 10.0.0-rc.11', stderr: '' }; + } + return { stdout: '', stderr: '' }; + }) as any; + + const log = makeLog(); + const result = await performNpmUpdateEdge('10.0.0-rc.12', '10.0.0-rc.11', log.fn); + expect(result).toBe('failed'); + expect(execCalls.map((call) => call.cmd)).toEqual([ + 'npm install -g @origintrail-official/dkg@10.0.0-rc.12', + 'dkg --version', + 'npm install -g @origintrail-official/dkg@10.0.0-rc.11', + 'dkg --version', + ]); + expect(log.calls.some((message) => message.includes('rollback restored'))).toBe(true); + }); + + it('rolls back on an exact-version mismatch, including semver prefix collisions', async () => { + let versionChecks = 0; + _autoUpdateIo.exec = (async (cmd: string, opts?: any) => { + execCalls.push({ cmd, opts }); + if (cmd === 'dkg --version') { + versionChecks += 1; + return versionChecks === 1 + ? { stdout: 'dkg 10.0.0-rc.12', stderr: '' } + : { stdout: 'dkg 10.0.0-rc.0', stderr: '' }; + } + return { stdout: '', stderr: '' }; + }) as any; + + const log = makeLog(); + const result = await performNpmUpdateEdge('10.0.0-rc.1', '10.0.0-rc.0', log.fn); + expect(result).toBe('failed'); + expect(execCalls.map((call) => call.cmd)).toEqual([ + 'npm install -g @origintrail-official/dkg@10.0.0-rc.1', + 'dkg --version', + 'npm install -g @origintrail-official/dkg@10.0.0-rc.0', + 'dkg --version', + ]); + expect(log.calls.some((message) => message.includes('expected 10.0.0-rc.1'))).toBe(true); + }); + it('surfaces a prefix-configuration advisory on EACCES', async () => { _autoUpdateIo.exec = (() => { const err: any = new Error("EACCES: permission denied, mkdir '/usr/local/lib/node_modules'"); diff --git a/scripts/devnet-test-issue-1567-global-cli-rollback.sh b/scripts/devnet-test-issue-1567-global-cli-rollback.sh new file mode 100755 index 0000000000..f35aa8e708 --- /dev/null +++ b/scripts/devnet-test-issue-1567-global-cli-rollback.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Host-level devnet regression for #1567. It exercises the real edge updater +# orchestration in an isolated npm prefix/PATH: the target install reports a +# semver-prefix collision (rc.12 for expected rc.1), so verification must reject +# it and the previous rc.0 CLI must be restored. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +tmp="$(mktemp -d "${TMPDIR:-/tmp}/dkg-1567.XXXXXX")" +trap 'rm -rf "$tmp"' EXIT INT TERM +mkdir -p "$tmp/bin" "$tmp/home" +version_file="$tmp/version" +printf '%s\n' '10.0.0-rc.0' >"$version_file" + +printf '%s\n' '#!/bin/sh' \ + 'case "$*" in' \ + ' *10.0.0-rc.1*) printf "%s\n" "10.0.0-rc.12" >"$FAKE_DKG_VERSION_FILE" ;;' \ + ' *10.0.0-rc.0*) printf "%s\n" "10.0.0-rc.0" >"$FAKE_DKG_VERSION_FILE" ;;' \ + ' *) exit 2 ;;' \ + 'esac' >"$tmp/bin/npm" +printf '%s\n' '#!/bin/sh' \ + 'printf "dkg %s\n" "$(cat "$FAKE_DKG_VERSION_FILE")"' >"$tmp/bin/dkg" +chmod +x "$tmp/bin/npm" "$tmp/bin/dkg" + +result="$( + PATH="$tmp/bin:$PATH" DKG_HOME="$tmp/home" FAKE_DKG_VERSION_FILE="$version_file" \ + node --input-type=module <<'NODE' +import { performNpmUpdateEdge } from './packages/cli/dist/daemon/auto-update.js'; +const logs = []; +const result = await performNpmUpdateEdge('10.0.0-rc.1', '10.0.0-rc.0', (line) => logs.push(line)); +process.stdout.write(JSON.stringify({ result, logs })); +NODE +)" + +RESULT="$result" node -e ' +const value = JSON.parse(process.env.RESULT); +if (value.result !== "failed") throw new Error(`expected failed-with-rollback, got ${value.result}`); +if (!value.logs.some((line) => line.includes("expected 10.0.0-rc.1"))) throw new Error("exact mismatch was not detected"); +if (!value.logs.some((line) => line.includes("rollback restored"))) throw new Error("rollback was not verified"); +' +[[ "$(cat "$version_file")" == 10.0.0-rc.0 ]] || { echo "[#1567] FAIL: previous CLI not restored" >&2; exit 1; } +[[ "$(cat "$tmp/home/previous-version")" == 10.0.0-rc.0 ]] || { echo "[#1567] FAIL: rollback target not recorded" >&2; exit 1; } +echo "[#1567] PASS: exact-version self-check rejected rc.12 and restored rc.0" From 62ab6231d2a03b1b6beb62117fb8ecaa0e70dc0f Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:13:35 +0200 Subject: [PATCH 09/12] Retire oxigraph-worker backend (#1539) * refactor(storage): retire oxigraph-worker backend * fix(cli): close store migration review gaps * test(storage): align main coverage with worker retirement * test(cli): isolate lifecycle wiring from store startup * fix(cli): address store retirement review * test(cli): make admission saturation deterministic * refactor(storage): centralize backend taxonomy * refactor(storage): clarify backend boundaries * test(cli): materialize managed store in startup mocks * fix(storage): separate adapter and daemon policy * fix(storage): enforce runtime backend capabilities * test(cli): pin operator config at request boundary * test(cli): wait for publisher readiness * refactor(cli): expose canonical route store views --------- Co-authored-by: Branimir Rakic --- CHANGELOG.md | 4 + README.md | 8 +- bench/store-read-latency.bench.ts | 188 +---- docs/use-dkg/storage-sparql-http.md | 4 +- packages/agent/src/dkg-agent-helpers.ts | 1 - packages/agent/src/dkg-agent-types.ts | 2 +- packages/agent/src/dkg-agent.ts | 4 +- .../agent/src/sync/responder/sync-handler.ts | 2 +- .../test/context-graph-discovery.test.ts | 6 +- packages/cli/src/commands/hermes.ts | 3 +- packages/cli/src/commands/init.ts | 10 +- packages/cli/src/commands/lifecycle.ts | 2 +- packages/cli/src/commands/mcp.ts | 3 +- packages/cli/src/commands/openclaw.ts | 3 +- packages/cli/src/config.ts | 57 +- packages/cli/src/daemon/chain-reset-wipe.ts | 129 +-- packages/cli/src/daemon/daemon-state.ts | 76 ++ packages/cli/src/daemon/handle-request.ts | 12 +- packages/cli/src/daemon/lifecycle.ts | 60 +- packages/cli/src/daemon/oxigraph-managed.ts | 5 +- packages/cli/src/daemon/routes/context.ts | 26 +- packages/cli/src/daemon/routes/status.ts | 46 +- packages/cli/src/daemon/store-runtime.ts | 174 +++++ packages/cli/src/publisher-runner.ts | 16 +- packages/cli/src/store-backends.ts | 226 ++++++ packages/cli/src/store-wizard.ts | 233 +++--- packages/cli/test/chain-reset-wipe.test.ts | 30 +- .../test/daemon-http-behavior-extra.test.ts | 2 +- .../cli/test/daemon-http-inflight-cap.test.ts | 171 ++-- .../test/daemon-startup-validation.test.ts | 190 ++++- packages/cli/test/daemon-state.test.ts | 53 ++ .../daemon-storage-ack-timing-wiring.test.ts | 2 + .../daemon-sync-agents-meta-wiring.test.ts | 1 + .../test/daemon/plugin-routes-api.e2e.test.ts | 2 +- .../handle-request-store-persistence.test.ts | 103 +++ packages/cli/test/helpers/live-daemon.ts | 13 +- packages/cli/test/oxigraph-managed.test.ts | 4 +- .../cli/test/publisher-managed-store.test.ts | 22 + packages/cli/test/publisher-wallets.test.ts | 42 + packages/cli/test/status-route-rpc.test.ts | 173 +++- .../cli/test/store-backend-taxonomy.test.ts | 122 +++ packages/cli/test/store-health-check.test.ts | 2 +- packages/cli/test/store-identity-tag.test.ts | 2 +- packages/cli/test/store-runtime.test.ts | 164 ++++ packages/cli/test/store-wizard.test.ts | 120 ++- .../cli/test/validate-store-config.test.ts | 24 +- .../test/write-preflight-resilience.test.ts | 41 +- packages/cli/vitest.unit.config.ts | 4 +- packages/core/src/proto/storage-ack.ts | 6 +- .../core/test/ensure-dkg-node-config.test.ts | 2 +- .../test/kafka-plugin-api.e2e.test.ts | 2 +- .../devnet/publish-ack-quorum.devnet.spec.ts | 2 +- .../write-preflight-guard.devnet.spec.ts | 4 +- packages/query/src/query-handler.ts | 2 +- packages/storage/README.md | 34 - .../src/adapters/oxigraph-worker-impl.ts | 18 - .../storage/src/adapters/oxigraph-worker.ts | 737 ------------------ packages/storage/src/adapters/oxigraph.ts | 4 +- packages/storage/src/index.ts | 23 +- packages/storage/src/store-backends.ts | 84 ++ packages/storage/src/triple-store.ts | 101 ++- .../test/graph-set-index-store.test.ts | 6 +- .../storage/test/is-external-backend.test.ts | 1 - .../test/oxigraph-worker-resilience.test.ts | 261 ------- .../test/oxigraph-worker-respawn.test.ts | 348 --------- packages/storage/test/storage.test.ts | 93 ++- scripts/devnet.sh | 8 +- scripts/publisher-smoke-test.sh | 2 +- 68 files changed, 2164 insertions(+), 2161 deletions(-) create mode 100644 packages/cli/src/daemon/daemon-state.ts create mode 100644 packages/cli/src/daemon/store-runtime.ts create mode 100644 packages/cli/src/store-backends.ts create mode 100644 packages/cli/test/daemon-state.test.ts create mode 100644 packages/cli/test/handle-request-store-persistence.test.ts create mode 100644 packages/cli/test/publisher-managed-store.test.ts create mode 100644 packages/cli/test/store-backend-taxonomy.test.ts create mode 100644 packages/cli/test/store-runtime.test.ts delete mode 100644 packages/storage/src/adapters/oxigraph-worker-impl.ts delete mode 100644 packages/storage/src/adapters/oxigraph-worker.ts create mode 100644 packages/storage/src/store-backends.ts delete mode 100644 packages/storage/test/oxigraph-worker-resilience.test.ts delete mode 100644 packages/storage/test/oxigraph-worker-respawn.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ceea71cf4..d283a829ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to the DKG V10 node are documented here. The format is based ## [Unreleased] +### Removed — `oxigraph-worker` backend support + +- **The embedded `oxigraph-worker` backend is retired.** The storage package no longer exports or registers the worker-thread adapter, `createTripleStore({ backend: "oxigraph-worker" })` fails with an actionable migration error, and CLI config validation refuses explicit `store.backend: "oxigraph-worker"` before daemon boot. Block-less configs now resolve to the daemon-managed `oxigraph-server` default; if a legacy `store.nq` file exists, daemon boot requires `DKG_ACCEPT_STORE_RESET=1` so operators acknowledge the fresh-store cutover. + ## [10.0.6] - 2026-07-10 Sync-, storage-, and admission-path hardening on top of 10.0.5, plus the StorageACK priority-lane follow-ups and a set of CLI/RPC fixes. Eliminates the perpetually-dirty graph-set-index full scan that was saturating managed Oxigraph cores, bounds sync-responder memory and coalesces duplicate sync fan-out, makes network admission probing back off and use canonical peer ids, and completes the StorageACK priority-lane hardening (ACK candidate selection, async promote-queue read serialization, and OT-RFC-49 host-mode ciphertext strip-by-curation). **No smart-contract changes — no deployment required** (no Solidity source, ABI, or mainnet/testnet deployment-registry changes since 10.0.5). diff --git a/README.md b/README.md index cb9a5aa15e..8fb7d724a0 100644 --- a/README.md +++ b/README.md @@ -403,11 +403,13 @@ analysis reports are under `bench/results/profiles/`, including ## Triple Store Backends -A DKG node keeps every assertion in an [RDF](https://www.w3.org/RDF/) triple store. Out of the box the node runs an embedded [Oxigraph](https://github.com/oxigraph/oxigraph) instance, which is everything you need on a workstation — no extra process, no extra port, no extra config. Heavier deployments can swap in [Blazegraph](https://blazegraph.com/) (the mainnet store) or any SPARQL 1.1 server. +A DKG node keeps every assertion in an [RDF](https://www.w3.org/RDF/) triple store. Out of the box the daemon manages a local [Oxigraph](https://github.com/oxigraph/oxigraph) server, so a workstation needs no separate setup. Heavier deployments can swap in [Blazegraph](https://blazegraph.com/) (the mainnet store) or any SPARQL 1.1 server. | Backend | When to pick it | |---|---| -| `oxigraph-worker` (default) | Single-operator nodes, dev, CI. No setup. File-backed, capped at process RAM. | +| `oxigraph-server` (default) | Single-operator nodes, dev, CI. Managed automatically by the daemon with persistent local storage. | +| `oxigraph` | Embedded in-memory Oxigraph for development and short-lived tests. | +| `oxigraph-persistent` | Embedded persistent Oxigraph when an explicit existing store path is required. | | `blazegraph` | High-throughput nodes, mainnet parity, very large graphs (10M+ quads). Run as a separate daemon (Docker or `java -jar`). Shares cleanly with V6 / V8 instances — DKG scopes its writes to the `did:dkg:context-graph:` named-graph prefix. | | `sparql-http` | Any SPARQL 1.1 Protocol server (Fuseki, GraphDB, Stardog, Neptune…). Bring your own URL + (optional) auth header. | @@ -420,7 +422,7 @@ Two paths: ``` $ dkg init … -Triple store backend (oxigraph / blazegraph) (oxigraph): blazegraph +Triple store backend (oxigraph-server / oxigraph / blazegraph) (oxigraph-server): blazegraph Blazegraph SPARQL endpoint URL: http://127.0.0.1:9999/bigdata/namespace/mynode/sparql Store endpoint reachable: blazegraph http://127.0.0.1:9999/bigdata/namespace/mynode/sparql ``` diff --git a/bench/store-read-latency.bench.ts b/bench/store-read-latency.bench.ts index 28d16edd52..9f2ec700ce 100644 --- a/bench/store-read-latency.bench.ts +++ b/bench/store-read-latency.bench.ts @@ -1,5 +1,3 @@ -import { existsSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; import { defineSuite } from 'esbench'; // Import the store classes + types from their SPECIFIC source modules rather // than the storage barrel: the barrel also re-exports GraphManager / @@ -8,7 +6,6 @@ import { defineSuite } from 'esbench'; // clean checkout. These adapter modules depend only on `oxigraph` + the local // triple-store types, so the bench needs nothing beyond the storage build. import { OxigraphStore } from '../packages/storage/src/adapters/oxigraph.ts'; -import { OxigraphWorkerStore } from '../packages/storage/src/adapters/oxigraph-worker.ts'; import type { Quad, QueryResult } from '../packages/storage/src/triple-store.ts'; import { GET_TOTAL_TRIPLES_SPARQL, parseRdfInt } from '../packages/cli/src/daemon/metrics-queries.ts'; import { benchAsyncWithHooks } from './support/esbench-case-hooks.ts'; @@ -24,33 +21,11 @@ import { benchAsyncWithHooks } from './support/esbench-case-hooks.ts'; * - the production `getTotalTriples` `COUNT(*)` aggregate the 30s metrics * collector runs (`packages/cli/src/daemon/lifecycle.ts`). * - * Backends (env `DKG_BENCH_STORE_BACKENDS`, comma-separated): - * - `inprocess` — `OxigraphStore`, no build step required. - * - `worker` — `OxigraphWorkerStore`, the PRODUCTION backend; requires - * `pnpm --filter @origintrail-official/dkg-storage build` first (it spawns a - * compiled worker artefact). - * Default: `inprocess`, plus `worker` automatically when its compiled artefact - * is present (so a built tree / CI measures both; an unbuilt tree degrades to - * `inprocess`-only instead of erroring). - * - * CONTENTION — the `(under write load)` cases — is measured ONLY on the `worker` - * backend, by design. The production read-starvation is a property of the - * single-writer oxigraph WORKER, whose message queue serialises a read behind - * queued writes — exactly what an out-of-process MVCC server (#938) relieves. - * The in-process `OxigraphStore` runs its insert/query work SYNCHRONOUSLY on one - * thread, so a read and a write can never truly overlap; a same-thread "writer" - * would only measure event-loop interleaving (not store contention) and could - * report misleadingly idle reads. So `inprocess` runs the idle read-latency - * baselines only. - * * Store sizes via env `DKG_BENCH_STORE_SIZES` (default `1k,50k`). */ -type Backend = 'inprocess' | 'worker'; - interface ReadStore { insert(quads: Quad[]): Promise; - delete(quads: Quad[]): Promise; query(sparql: string): Promise; close(): Promise; } @@ -63,13 +38,6 @@ const READ_LIMIT1 = `SELECT ?s WHERE { GRAPH <${GRAPH}> { ?s ?p ?o } } LIMIT 1`; // synthetic data lives in a named graph, so the `GRAPH ?g` branch carries the scan. const READ_TOTAL_TRIPLES = GET_TOTAL_TRIPLES_SPARQL; -// Bounded write churn: the background writer repeatedly inserts then deletes a -// fixed batch in a region disjoint from the pre-populated base, so it generates -// sustained write work WITHOUT drifting the store size (which would otherwise -// confound the getTotalTriples-under-load measurement). -const CHURN_BATCH = 50; -const CHURN_OFFSET = 1_000_000_000; - const STORE_SIZES: Record = { '1k': 1_000, '10k': 10_000, '50k': 50_000, '200k': 200_000 }; const INSERT_CHUNK = 1_000; @@ -111,46 +79,6 @@ function makeQuads(count: number, offset: number): Quad[] { return quads; } -// The compiled worker artefact sits next to the storage dist build; its presence -// means the `worker` backend can be constructed without throwing. -function workerArtifactAvailable(): boolean { - try { - return existsSync(fileURLToPath(new URL('../packages/storage/dist/adapters/oxigraph-worker-impl.js', import.meta.url))); - } catch { - return false; - } -} - -function resolveBackends(): Backend[] { - const raw = process.env.DKG_BENCH_STORE_BACKENDS?.trim(); - if (!raw) { - if (workerArtifactAvailable()) return ['inprocess', 'worker']; - // Loud, because the worker backend is the one this regression is about — a - // silent inprocess-only run would look like it measured the production path. - console.warn( - '[store-read-latency] worker backend SKIPPED (compiled adapter missing) — ' + - 'measuring `inprocess` idle read latency ONLY, NOT the production write-contention path. ' + - 'Run `pnpm --filter @origintrail-official/dkg-storage build` (or `pnpm bench:store-read`, ' + - 'which builds it) to include the worker backend.', - ); - return ['inprocess']; - } - const known = new Set(['inprocess', 'worker']); - const requested = raw.split(',').map((p) => p.trim().toLowerCase()).filter(Boolean); - for (const b of requested) { - if (!known.has(b as Backend)) { - throw new Error(`Unknown DKG_BENCH_STORE_BACKENDS entry "${b}". Expected: inprocess, worker`); - } - } - if (requested.includes('worker') && !workerArtifactAvailable()) { - throw new Error( - 'DKG_BENCH_STORE_BACKENDS requested "worker", but the compiled adapter is missing. ' + - 'Run `pnpm --filter @origintrail-official/dkg-storage build` first.', - ); - } - return requested.length > 0 ? (requested as Backend[]) : ['inprocess']; -} - function resolveStoreSizeLabels(): string[] { const raw = process.env.DKG_BENCH_STORE_SIZES?.trim(); const labels = raw ? raw.split(',').map((p) => p.trim().toLowerCase()).filter(Boolean) : ['1k', '50k']; @@ -162,15 +90,8 @@ function resolveStoreSizeLabels(): string[] { return labels; } -function createStore(backend: Backend): ReadStore { - // `worker` availability is gated in resolveBackends(), so by the time we get - // here the compiled adapter is present. - return backend === 'worker' ? new OxigraphWorkerStore() : new OxigraphStore(); -} - export default defineSuite({ params: { - backend: resolveBackends(), storeSize: resolveStoreSizeLabels(), }, baseline: { @@ -185,63 +106,20 @@ export default defineSuite({ warmup: 1, }, async setup(scene) { - const backend = scene.params.backend as Backend; const sizeLabel = scene.params.storeSize as string; const quadCount = STORE_SIZES[sizeLabel]; - const store = createStore(backend); + const store: ReadStore = new OxigraphStore(); - const churn = makeQuads(CHURN_BATCH, CHURN_OFFSET); - let writerActive = false; - let writerDone: Promise | undefined; - let writerError: unknown; - - const stopWriter = async (): Promise => { - if (!writerDone) return; - writerActive = false; - const done = writerDone; - writerDone = undefined; - // The loop captures its own failures into `writerError`, so this never rejects. - await done; - // Remove the churn batch so a half-applied cycle (an insert without its - // matching delete) can't leak into the next iteration's getTotalTriples count. - try { - await store.delete(churn); - } catch { - /* store may be mid-teardown */ - } - // Fail fast if the writer died: a stopped writer must never let an - // `(under write load)` case record a successful (effectively idle) sample. - if (writerError !== undefined) { - const err = writerError; - writerError = undefined; - throw new Error(`background writer died during the under-load iteration: ${errorText(err)}`); - } - }; - - // ONE ordered teardown for the scene: stop the writer and await its loop - // BEFORE closing the store, so an in-flight insert/delete can never race a - // closed store/worker. esbench may run scene teardown hooks concurrently, so - // all ordering lives inside this single callback. Registered up-front so the - // store is still closed (and any worker thread terminated) even if the - // population below throws. + // Registered up-front so the store is still closed even if population or a + // benchmark case throws. scene.teardown(async () => { - let stopErr: unknown; - try { - await stopWriter(); - } catch (err) { - stopErr = err; // capture; still close the store below - } - // `close()` is the only place the worker thread is joined, so a failure - // here is a real teardown bug — surface it (log + reject) rather than - // swallow it and let a broken worker benchmark look green. try { await store.close(); } catch (closeErr) { - console.error(`[store-read-latency] store.close() failed (worker thread may not have joined): ${errorText(closeErr)}`); + console.error(`[store-read-latency] store.close() failed: ${errorText(closeErr)}`); throw closeErr; } - if (stopErr !== undefined) throw stopErr; }); // Pre-populate the base graph the reads scan. @@ -256,63 +134,5 @@ export default defineSuite({ benchAsyncWithHooks(scene, 'read getTotalTriples (idle)', async () => { assertCountAtLeast(await store.query(READ_TOTAL_TRIPLES), quadCount, 'read getTotalTriples'); }, {}); - - // Contention is meaningful only on the worker backend (see file docstring): - // the in-process store is single-threaded + synchronous, so reads and writes - // cannot truly overlap. Skip the `(under write load)` cases for it rather - // than report a misleading same-thread number. - if (backend !== 'worker') return; - - // Background writer for the worker-backend `(under write load)` cases, - // scoped to each loaded iteration (not to case-execution order): - // - `beforeIteration` (startWriter) AWAITS one full insert/delete cycle so - // writes are provably queued in the worker before the measured read. - // - a writer that dies is recorded in `writerError` and surfaced as a - // FAILED case — startWriter throws if it died before the first cycle, - // the workload throws if it died mid-measurement, and stopWriter throws - // if it died just after — so a broken writer can never silently degrade - // into an idle read. - // - `afterIteration` (stopWriter) stops it and clears the churn batch. - const startWriter = async (): Promise => { - if (writerActive) return; - writerActive = true; - writerError = undefined; - let signalFirstCycle!: () => void; - const firstCycle = new Promise((resolve) => { signalFirstCycle = resolve; }); - writerDone = (async () => { - try { - let signalled = false; - while (writerActive) { - await store.insert(churn); - await store.delete(churn); - if (!signalled) { signalled = true; signalFirstCycle(); } - } - } catch (err) { - writerError = err; - } finally { - // Unblock the barrier even if the first cycle threw, so a writer - // failure surfaces (below) instead of hanging the benchmark. - signalFirstCycle(); - } - })(); - await firstCycle; - if (writerError !== undefined) { - throw new Error(`background writer failed before its first write cycle: ${errorText(writerError)}`); - } - }; - - benchAsyncWithHooks(scene, 'read LIMIT 1 (under write load)', async () => { - assertNonEmptySelect(await store.query(READ_LIMIT1), 'read LIMIT 1'); - if (writerError !== undefined) { - throw new Error(`background writer died during the measurement: ${errorText(writerError)}`); - } - }, { beforeIteration: startWriter, afterIteration: stopWriter }); - - benchAsyncWithHooks(scene, 'read getTotalTriples (under write load)', async () => { - assertCountAtLeast(await store.query(READ_TOTAL_TRIPLES), quadCount, 'read getTotalTriples'); - if (writerError !== undefined) { - throw new Error(`background writer died during the measurement: ${errorText(writerError)}`); - } - }, { beforeIteration: startWriter, afterIteration: stopWriter }); }, }); diff --git a/docs/use-dkg/storage-sparql-http.md b/docs/use-dkg/storage-sparql-http.md index 9d70af2f93..3a8ea2c5b8 100644 --- a/docs/use-dkg/storage-sparql-http.md +++ b/docs/use-dkg/storage-sparql-http.md @@ -7,7 +7,7 @@ doc_type: how-to # Using an external SPARQL store (Oxigraph server, etc.) -The DKG node can use any **SPARQL 1.1 Protocol**–compliant store you run yourself, instead of its default daemon-managed local Oxigraph server (or the embedded `oxigraph-worker` fallback). That gives you: +The DKG node can use any **SPARQL 1.1 Protocol**–compliant store you run yourself, instead of its default daemon-managed local Oxigraph server. That gives you: - **Real on-disk persistence** (e.g. Oxigraph server with RocksDB) - **Larger graphs** without holding everything in the Node process @@ -118,4 +118,4 @@ await agent.start(); New installs default to a **daemon-managed local Oxigraph server** (`store.backend: "oxigraph-server"`): `dkg init`, `dkg openclaw/hermes/mcp setup`, or accepting the wizard default writes this block. The daemon fetches the pinned `oxigraph` binary on first boot and runs it on loopback, giving MVCC concurrent reads and incremental RocksDB persistence. -If a config has **no** `store` block at all, the runtime falls back to the embedded in-process **`oxigraph-worker`** (a single-writer store that rewrites its on-disk N-Quads dump under `dataDir` on every flush) — fine for development and small nodes. For very large graphs or existing infrastructure, use `sparql-http` with an external store. +If a config has **no** `store` block at all, the runtime now uses the same daemon-managed **`oxigraph-server`** default. The old embedded **`oxigraph-worker`** backend has been retired; configs that still name it fail fast with a migration message. For very large graphs or existing infrastructure, use `sparql-http` with an external store. diff --git a/packages/agent/src/dkg-agent-helpers.ts b/packages/agent/src/dkg-agent-helpers.ts index 3d53b4157a..5e7a39b814 100644 --- a/packages/agent/src/dkg-agent-helpers.ts +++ b/packages/agent/src/dkg-agent-helpers.ts @@ -403,7 +403,6 @@ export function applyDefaultLargeLiteralStorage( export function isLocalOxigraphConfig(storeConfig: TripleStoreConfig): boolean { return storeConfig.backend === 'oxigraph' - || storeConfig.backend === 'oxigraph-worker' || storeConfig.backend === 'oxigraph-persistent'; } diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index 7b3825d16e..a64cd160a7 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -923,7 +923,7 @@ export interface DKGAgentConfig { }>; dataDir?: string; store?: TripleStore; - /** Triple store backend configuration (e.g. oxigraph-worker, blazegraph). If omitted, defaults to oxigraph-worker when dataDir is set. */ + /** Triple store backend configuration (e.g. oxigraph-server runtime view, oxigraph-persistent, blazegraph). If omitted, dataDir agents use oxigraph-persistent. */ storeConfig?: TripleStoreConfig; /** Out-of-line storage for large public SWM RDF literal object terms. Defaults on for local Oxigraph-backed dataDir stores. */ largeLiteralStorage?: LargeLiteralStorageConfig; diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index f27f95994a..7d360f6c8b 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -685,11 +685,11 @@ export class DKGAgent extends DKGAgentBase { const { join } = await import('node:path'); const persistPath = join(config.dataDir, 'store.nq'); store = await createTripleStore({ - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: persistPath }, largeLiteralStorage: defaultLargeLiteralStorage(config.dataDir, config.largeLiteralStorage), }); - log.info(ctx, `Persistent triple store (worker thread): ${persistPath}`); + log.info(ctx, `Persistent triple store: ${persistPath}`); } else { store = await createTripleStore({ backend: 'oxigraph' }); log.warn(ctx, `No dataDir — triple store is in-memory (data will be lost on restart)`); diff --git a/packages/agent/src/sync/responder/sync-handler.ts b/packages/agent/src/sync/responder/sync-handler.ts index 646afe3515..1ea7142003 100644 --- a/packages/agent/src/sync/responder/sync-handler.ts +++ b/packages/agent/src/sync/responder/sync-handler.ts @@ -447,7 +447,7 @@ export function registerSyncHandler(params: RegisterSyncHandlerParams): void { warnedPreDispatchCancellation = true; logWarn( createOperationContext('sync'), - 'Sync responder is using a store backend whose query AbortSignal is pre-dispatch only; in-flight sync queries cannot release responder capacity until the synchronous store call returns. Use oxigraph-worker or an HTTP SPARQL backend for interruptible long-query cancellation.', + 'Sync responder is using a store backend whose query AbortSignal is pre-dispatch only; in-flight sync queries cannot release responder capacity until the synchronous store call returns. Use an HTTP SPARQL backend for interruptible long-query cancellation.', ); } if (isWorkspace) { diff --git a/packages/agent/test/context-graph-discovery.test.ts b/packages/agent/test/context-graph-discovery.test.ts index bda11d560d..33429c681e 100644 --- a/packages/agent/test/context-graph-discovery.test.ts +++ b/packages/agent/test/context-graph-discovery.test.ts @@ -1303,8 +1303,10 @@ describe('listContextGraphs merge', () => { }, 15000); it('bypasses list cache for unknown configured store backends', async () => { - const backend = 'test-remote-list-cache-backend'; - registerTripleStoreAdapter(backend, async () => new OxigraphStore()); + const backend = registerTripleStoreAdapter( + 'test-remote-list-cache-backend', + async () => new OxigraphStore(), + ); const created = await DKGAgent.create({ kaNumberAllocator: makeTestKaNumberAllocator(), name: 'ContextGraphTestAgent', diff --git a/packages/cli/src/commands/hermes.ts b/packages/cli/src/commands/hermes.ts index 1d95782816..7f98eb0359 100644 --- a/packages/cli/src/commands/hermes.ts +++ b/packages/cli/src/commands/hermes.ts @@ -32,6 +32,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -189,7 +190,7 @@ hermesCmd ) .option( '--store ', - 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL and persists the store block after setup.', + `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL and persists the store block after setup.`, ) .option( '--store-url ', diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 4bd216b3fe..d49ad22b9b 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -36,6 +36,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -216,7 +217,7 @@ program ) .option( '--store ', - 'Pre-fill the triple-store backend prompt (oxigraph | blazegraph | sparql-http).', + `Pre-fill the triple-store backend prompt (${storeFlagBackendList(' | ')}).`, ) .option( '--store-url ', @@ -534,9 +535,8 @@ program chain: isNetworkSwitch ? chainSection : (chainSection ?? existing.chain), auth: { enabled: enableAuth, tokens: existing.auth?.tokens }, // Persist the chosen backend. `storeBlock === null` from the - // wizard means "use the local default" — we explicitly clear any - // existing block so re-running `dkg init` to switch from - // blazegraph back to oxigraph actually applies. + // wizard means "leave the store block omitted"; daemon boot treats + // that as the managed `oxigraph-server` default. store: storeBlock ?? undefined, }; await saveConfig(config); @@ -573,7 +573,7 @@ program const endpoint = o?.url ?? o?.queryEndpoint; return `${storeBlock.backend}${endpoint ? ` (${endpoint})` : ''}`; })() - : 'oxigraph (local default)' + : 'oxigraph-server (default)' }`, ); { diff --git a/packages/cli/src/commands/lifecycle.ts b/packages/cli/src/commands/lifecycle.ts index 5dace2f9ba..c4c8723c68 100644 --- a/packages/cli/src/commands/lifecycle.ts +++ b/packages/cli/src/commands/lifecycle.ts @@ -290,7 +290,7 @@ program // remote store. Quad count = null is rare in practice (cached // every 30 s on the daemon side) so when it shows up the // operator should treat it as an alert, not a no-op. - const backend = s.storeBackend ?? 'oxigraph-worker'; + const backend = s.storeBackend ?? 'oxigraph-server'; if (s.storeUrl) { const quads = s.storeQuads == null ? 'UNREACHABLE' : `${s.storeQuads.toLocaleString()} quads`; console.log(` Store: ${backend} (${s.storeUrl}) — ${quads}`); diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts index 9ff4c39d9c..d11ad64eaf 100644 --- a/packages/cli/src/commands/mcp.ts +++ b/packages/cli/src/commands/mcp.ts @@ -29,6 +29,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -157,7 +158,7 @@ mcpCmd .option('--yes', 'Auto-confirm per-client registrations (default false: prompt interactively in TTY mode; non-TTY auto-confirms — pass `--yes` in scripts for the safer scripted-environment posture)') .option( '--store ', - 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL and persists the store block after setup.', + `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL and persists the store block after setup.`, ) .option( '--store-url ', diff --git a/packages/cli/src/commands/openclaw.ts b/packages/cli/src/commands/openclaw.ts index 1d23846f7e..d0fa61b47c 100644 --- a/packages/cli/src/commands/openclaw.ts +++ b/packages/cli/src/commands/openclaw.ts @@ -32,6 +32,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -127,7 +128,7 @@ openclawCmd ) .option( '--store ', - 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL via an ASK probe and persists the store block after setup completes.', + `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL via an ASK probe and persists the store block after setup completes.`, ) .option( '--store-url ', diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index 6e4fa09152..dd2baa784a 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -19,6 +19,12 @@ import { STORAGE_ACK_TIMING_SAFETY_MARGIN_MS, type StorageAckTiming, } from '@origintrail-official/dkg-publisher'; +import { + getStoreBackendPolicy, + isExternalStoreBackend, + isRetiredStoreBackend, + configBackendList, +} from './store-backends.js'; /** * Per-step build timeouts (milliseconds) used by the git-based auto-update @@ -574,7 +580,7 @@ export interface DkgConfig { llm?: LlmConfig; /** Block explorer URL for TX links (default: derived from chainId). */ blockExplorerUrl?: string; - /** Triple store backend override (default: oxigraph-worker with file persistence). */ + /** Triple store backend override (default: daemon-managed oxigraph-server). */ store?: { backend: string; options?: Record; graphSetIndex?: boolean | GraphSetIndexConfig; changelog?: boolean }; /** * Intentional cap on how many persisted context-graph subscriptions a node @@ -1831,34 +1837,31 @@ export interface StoreConfigValidationError { export function validateStoreConfig(config: DkgConfig): StoreConfigValidationError[] { const errors: StoreConfigValidationError[] = []; const backend = config.store?.backend; - // Mirror of `isExternalBackend` from @origintrail-official/dkg-storage. - // Duplicated here to keep config.ts free of upward dependencies on the - // storage package (config.ts is leaf-imported by many other modules). - const isExternal = backend === 'blazegraph' || backend === 'sparql-http'; - if (!isExternal) return errors; + if (isRetiredStoreBackend(backend)) { + return [{ + field: 'store.backend', + message: + `${EXTERNAL_VALIDATION_PREFIX} "${backend}" is no longer supported. ` + + `Use one of: ${configBackendList()}.`, + }]; + } + if (!isExternalStoreBackend(backend)) return errors; const opts = (config.store?.options ?? {}) as Record; - - if (backend === 'blazegraph') { - if (typeof opts.url !== 'string' || !opts.url.trim()) { - errors.push({ - field: 'store.options.url', - message: - `${EXTERNAL_VALIDATION_PREFIX} is "blazegraph" but ` + - `store.options.url is missing. Set it to the SPARQL endpoint URL ` + - `(e.g. http://127.0.0.1:9999/bigdata/namespace/mynode/sparql) or ` + - `switch backend to oxigraph-worker.`, - }); - } - } else if (backend === 'sparql-http') { - if (typeof opts.queryEndpoint !== 'string' || !opts.queryEndpoint.trim()) { - errors.push({ - field: 'store.options.queryEndpoint', - message: - `${EXTERNAL_VALIDATION_PREFIX} is "sparql-http" but ` + - `store.options.queryEndpoint is missing. Set it to the SPARQL query URL.`, - }); - } + const policy = getStoreBackendPolicy(backend); + if (!policy || policy.kind !== 'external') { + throw new Error(`Missing external-store policy for "${backend}"`); + } + const queryOption = policy.queryEndpointOption; + const queryEndpoint = opts[queryOption]; + if (typeof queryEndpoint !== 'string' || !queryEndpoint.trim()) { + errors.push({ + field: `store.options.${queryOption}`, + message: + `${EXTERNAL_VALIDATION_PREFIX} is "${backend}" but ` + + `store.options.${queryOption} is missing. Set it to the SPARQL query endpoint URL ` + + `or switch backend to oxigraph-server.`, + }); } if (config.largeLiteralStorage?.enabled === true) { diff --git a/packages/cli/src/daemon/chain-reset-wipe.ts b/packages/cli/src/daemon/chain-reset-wipe.ts index 2bb6df3a2c..6d56c8bde4 100644 --- a/packages/cli/src/daemon/chain-reset-wipe.ts +++ b/packages/cli/src/daemon/chain-reset-wipe.ts @@ -64,8 +64,6 @@ */ import { existsSync, - readFileSync, - writeFileSync, readdirSync, rmSync, renameSync, @@ -74,32 +72,14 @@ import { } from 'node:fs'; import { join } from 'node:path'; import { isExternalBackend, getSparqlEndpoint, CHANGELOG_GRAPH } from '@origintrail-official/dkg-storage'; - -const STATE_FILE = '.network-state.json'; - -interface PersistedNetworkState { - /** Last chainResetMarker value the daemon booted on. */ - chainResetMarker: string | null; - /** - * Last triple-store backend the daemon booted on. Used by - * `detectBackendSwitch` to warn loudly when an operator hand-edits - * `config.store.backend` between boots — the new backend is fresh - * and empty, so silently booting would mean stale SWM/VM data is - * inaccessible. `null` on legacy state files (pre-RFC 120) and on - * first boot. (RFC 120 review point #6.) - */ - lastBackend?: string | null; - /** - * Last resolved network the daemon booted on (the `networkConfig` overlay - * name, e.g. `mainnet-gnosis`/`testnet`). Used by `detectNetworkSwitch` to - * abort boot when an operator repoints `config.networkConfig` at a different - * network on an existing data dir — the store holds the old network's - * chain-derived state (KC ids, merkle roots), which is meaningless on the - * new chain. `null`/absent on legacy state files and on first boot. - */ - lastNetworkConfig?: string | null; - savedAt: number; -} +import { + readPersistedDaemonState, + readPersistedNetworkConfig, + readPersistedStoreBackend, + writePersistedChainResetMarker, + writePersistedNetworkConfig, + writePersistedStoreBackend, +} from './daemon-state.js'; /** * Subset of `DkgConfig['store']` used by the wipe step to talk to an @@ -229,69 +209,6 @@ export function skipChainResetWipe(env: NodeJS.ProcessEnv = process.env): boolea return env.DKG_SKIP_CHAIN_RESET_WIPE === '1'; } -function loadState(dataDir: string): PersistedNetworkState | null { - try { - const raw = readFileSync(join(dataDir, STATE_FILE), 'utf8'); - const obj = JSON.parse(raw) as PersistedNetworkState; - if (typeof obj?.chainResetMarker !== 'string' && obj?.chainResetMarker !== null) return null; - return obj; - } catch { - return null; - } -} - -function saveState(dataDir: string, marker: string | null): void { - // Preserve any sibling fields (lastBackend) that `detectBackendSwitch` - // may have written. Otherwise a chain-reset wipe would clobber a - // freshly-recorded backend tag and the next boot would re-warn. - const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; - writeFileSync( - join(dataDir, STATE_FILE), - JSON.stringify( - { - ...existing, - chainResetMarker: marker, - savedAt: Date.now(), - } satisfies PersistedNetworkState, - null, - 2, - ), - ); -} - -function saveBackendTag(dataDir: string, backend: string): void { - const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; - writeFileSync( - join(dataDir, STATE_FILE), - JSON.stringify( - { - ...existing, - lastBackend: backend, - savedAt: Date.now(), - } satisfies PersistedNetworkState, - null, - 2, - ), - ); -} - -function saveNetworkTag(dataDir: string, networkConfig: string): void { - // Preserve sibling fields (chainResetMarker, lastBackend) like saveBackendTag. - const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; - writeFileSync( - join(dataDir, STATE_FILE), - JSON.stringify( - { - ...existing, - lastNetworkConfig: networkConfig, - savedAt: Date.now(), - } satisfies PersistedNetworkState, - null, - 2, - ), - ); -} - /** * Wipe the V10 data sitting in an external SPARQL endpoint. Runs after * the local file wipe so we don't strand the operator with a wiped FS @@ -582,7 +499,7 @@ export async function chainResetWipe( return { wiped: false, skipped: false, prevMarker: null, removedFiles: [], backedUpFiles: [], failedFiles: [] }; } - const prev = loadState(opts.dataDir); + const prev = readPersistedDaemonState(opts.dataDir); const prevMarker = prev?.chainResetMarker ?? null; if (prevMarker === opts.currentMarker) { @@ -664,7 +581,7 @@ export async function chainResetWipe( if (failedFiles.length === 0) { try { - saveState(opts.dataDir, opts.currentMarker); + writePersistedChainResetMarker(opts.dataDir, opts.currentMarker); markerPersisted = true; } catch (err) { log( @@ -718,7 +635,7 @@ export interface BackendSwitchDetectOptions { /** * Backend name from the current config. Pass the effective value * including the default — e.g. when `config.store?.backend` is - * undefined, callers should pass `'oxigraph-worker'` so the check + * undefined, callers should pass `'oxigraph-server'` so the check * is symmetric across "no store block" ↔ "explicit store block". */ currentBackend: string; @@ -750,21 +667,17 @@ export function detectBackendSwitch( opts: BackendSwitchDetectOptions, ): BackendSwitchDetectResult { const log = opts.log ?? (() => {}); - const prev = loadState(opts.dataDir); - const previous = - typeof prev?.lastBackend === 'string' && prev.lastBackend.length > 0 - ? prev.lastBackend - : null; + const previous = readPersistedStoreBackend(opts.dataDir); // First boot or legacy state file: silently record and move on. We - // explicitly do NOT treat null-previous as a "switch from - // oxigraph-worker"; that would re-warn every operator who upgrades + // explicitly do NOT treat null-previous as a "switch from the old + // implicit backend"; that would re-warn every operator who upgrades // into this release without ever having touched their store // configuration. Only operator-visible config changes between two // recorded backends count as a switch. if (previous === null) { try { - saveBackendTag(opts.dataDir, opts.currentBackend); + writePersistedStoreBackend(opts.dataDir, opts.currentBackend); } catch { // Non-fatal: if we can't write the tag now, we'll try again next // boot. The downside is one missed early-warning window. @@ -801,7 +714,7 @@ export function detectBackendSwitch( log(``); log(`DKG_ACCEPT_STORE_RESET=1 set — proceeding with the new backend.`); try { - saveBackendTag(opts.dataDir, opts.currentBackend); + writePersistedStoreBackend(opts.dataDir, opts.currentBackend); } catch (err) { log(`WARN: failed to persist new backend tag: ${(err as Error).message}. Will re-warn on next boot.`); } @@ -851,11 +764,7 @@ export function detectNetworkSwitch( opts: NetworkSwitchDetectOptions, ): NetworkSwitchDetectResult { const log = opts.log ?? (() => {}); - const prev = loadState(opts.dataDir); - const previous = - typeof prev?.lastNetworkConfig === 'string' && prev.lastNetworkConfig.length > 0 - ? prev.lastNetworkConfig - : null; + const previous = readPersistedNetworkConfig(opts.dataDir); // First boot or legacy state file: silently record and move on. We do NOT // treat null-previous as a switch — that would abort every operator who @@ -866,7 +775,7 @@ export function detectNetworkSwitch( // are caught normally. if (previous === null) { try { - saveNetworkTag(opts.dataDir, opts.currentNetworkConfig); + writePersistedNetworkConfig(opts.dataDir, opts.currentNetworkConfig); } catch { // Non-fatal: retry the tag write next boot. } @@ -906,7 +815,7 @@ export function detectNetworkSwitch( log(``); log(`DKG_ACCEPT_NETWORK_SWITCH=1 set — proceeding on the new network.`); try { - saveNetworkTag(opts.dataDir, opts.currentNetworkConfig); + writePersistedNetworkConfig(opts.dataDir, opts.currentNetworkConfig); } catch (err) { log(`WARN: failed to persist new network tag: ${(err as Error).message}. Will re-warn on next boot.`); } diff --git a/packages/cli/src/daemon/daemon-state.ts b/packages/cli/src/daemon/daemon-state.ts new file mode 100644 index 0000000000..66d546821e --- /dev/null +++ b/packages/cli/src/daemon/daemon-state.ts @@ -0,0 +1,76 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const DAEMON_STATE_FILE = '.network-state.json'; + +/** Persisted boot state shared by reset and configuration-switch guards. */ +export interface PersistedDaemonState { + chainResetMarker: string | null; + lastBackend?: string | null; + lastNetworkConfig?: string | null; + savedAt: number; +} + +export function readPersistedDaemonState(dataDir: string): PersistedDaemonState | null { + try { + const raw = readFileSync(join(dataDir, DAEMON_STATE_FILE), 'utf8'); + const state = JSON.parse(raw) as PersistedDaemonState; + if ( + typeof state?.chainResetMarker !== 'string' + && state?.chainResetMarker !== null + ) { + return null; + } + return state; + } catch { + return null; + } +} + +function updatePersistedDaemonState( + dataDir: string, + patch: Partial>, +): void { + const existing = readPersistedDaemonState(dataDir) ?? { + chainResetMarker: null, + savedAt: 0, + }; + writeFileSync( + join(dataDir, DAEMON_STATE_FILE), + JSON.stringify( + { + ...existing, + ...patch, + savedAt: Date.now(), + } satisfies PersistedDaemonState, + null, + 2, + ), + ); +} + +export function readPersistedStoreBackend(dataDir: string): string | null { + const state = readPersistedDaemonState(dataDir); + return typeof state?.lastBackend === 'string' && state.lastBackend.length > 0 + ? state.lastBackend + : null; +} + +export function readPersistedNetworkConfig(dataDir: string): string | null { + const state = readPersistedDaemonState(dataDir); + return typeof state?.lastNetworkConfig === 'string' && state.lastNetworkConfig.length > 0 + ? state.lastNetworkConfig + : null; +} + +export function writePersistedChainResetMarker(dataDir: string, marker: string | null): void { + updatePersistedDaemonState(dataDir, { chainResetMarker: marker }); +} + +export function writePersistedStoreBackend(dataDir: string, backend: string): void { + updatePersistedDaemonState(dataDir, { lastBackend: backend }); +} + +export function writePersistedNetworkConfig(dataDir: string, networkConfig: string): void { + updatePersistedDaemonState(dataDir, { lastNetworkConfig: networkConfig }); +} diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index 4297ac48c3..918080a9f8 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -311,7 +311,12 @@ import { reverseLocalAgentSetupForUi, refreshLocalAgentIntegrationFromUi, } from './local-agents.js'; -import type { MemoryGraphChangedEvent, NotificationSseEvent, RequestContext } from './routes/context.js'; +import { + createRequestStoreContext, + type MemoryGraphChangedEvent, + type NotificationSseEvent, + type RequestContext, +} from './routes/context.js'; import { handleStatusRoutes } from './routes/status.js'; import { handleAgentChatRoutes } from './routes/agent-chat.js'; import { handleOpenclawRoutes } from './routes/openclaw.js'; @@ -330,6 +335,7 @@ import { handleOperationalWalletRoutes } from './routes/operational-wallets.js'; import { handleNotificationRoutes } from './routes/notifications.js'; import { handlePluginRoutes } from './routes/plugins.js'; import type { RoutePlugin } from './plugin-api.js'; +import type { StoreRuntimeContext } from './store-runtime.js'; export async function handleRequest( @@ -338,7 +344,7 @@ export async function handleRequest( agent: DKGAgent, publisherControl: ReturnType, publisherRuntime: PublisherRuntime | null, - config: DkgConfig, + storeRuntime: StoreRuntimeContext, startedAt: number, dashDb: DashboardDB, opWallets: import("@origintrail-official/dkg-agent").OpWalletsConfig, @@ -383,7 +389,7 @@ export async function handleRequest( publisherControl, publisherRuntime, publisherAvailability, - config, + ...createRequestStoreContext(storeRuntime), startedAt, dashDb, opWallets, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 6aad669619..1fa6438cac 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -40,7 +40,7 @@ import { import { execSync, exec, execFile } from "node:child_process"; import { promisify } from "node:util"; import { join, dirname, resolve } from 'node:path'; -import { existsSync, readdirSync, readFileSync, openSync, closeSync, writeFileSync as fsWriteFileSync, unlinkSync } from 'node:fs'; +import { readdirSync, readFileSync, openSync, closeSync, writeFileSync as fsWriteFileSync, unlinkSync } from 'node:fs'; // Namespace import: our Phase-8 install-context builder (~line 290) calls // `osModule.homedir()`, and the later agent-identity probe (~line 6851) // uses `osModule.hostname()` + `osModule.userInfo()`. v10-rc's new @@ -306,6 +306,10 @@ import { } from './store-health-check.js'; import { startManagedOxigraph } from './oxigraph-managed.js'; import type { OxigraphServerHandle } from './oxigraph-server.js'; +import { + resolveDaemonStoreBootPlan, + resolveDaemonStoreRuntime, +} from './store-runtime.js'; import { resetNatStatus, startNatStatusWatcher } from './nat-status.js'; import { OPENCLAW_UI_CONNECT_TIMEOUT_MS, @@ -1127,6 +1131,25 @@ export async function runDaemonInner( ...resolveNetworkDefaultContextGraphs(network), ]), ]; + const acceptStoreReset = process.env.DKG_ACCEPT_STORE_RESET === '1'; + const storeDecision = resolveDaemonStoreBootPlan({ + config, + dataDir: dkgDir(), + acceptStoreReset, + }); + + if (storeDecision.kind === 'invalid-config') { + exitOnStoreConfigErrors(storeDecision.operatorConfig, log); + throw new Error('Invalid store config validation unexpectedly returned'); + } + if (storeDecision.kind === 'blocked-legacy-cutover') { + log(storeDecision.message); + process.exit(1); + } + const storeBoot = storeDecision; + if (storeBoot.notice) { + log(storeBoot.notice); + } // Auto-wipe per-node chain-state derived files (oxigraph store, publish // journal, random-sampling WAL) when the maintainer bumps @@ -1143,8 +1166,8 @@ export async function runDaemonInner( // silently would look like data loss to the operator. const backendSwitch = detectBackendSwitch({ dataDir: dkgDir(), - currentBackend: config.store?.backend ?? 'oxigraph-worker', - acceptStoreReset: process.env.DKG_ACCEPT_STORE_RESET === '1', + currentBackend: storeBoot.effectiveStore.backend, + acceptStoreReset, log, }); if (backendSwitch.aborted) { @@ -1180,7 +1203,7 @@ export async function runDaemonInner( let managedOxigraph: OxigraphServerHandle | null = null; let managed: Awaited> = null; try { - managed = await startManagedOxigraph({ config, dataDir: dkgDir(), log }); + managed = await startManagedOxigraph({ config: storeBoot.effectiveConfig, dataDir: dkgDir(), log }); if (managed) { managedOxigraph = managed.handle; // Every remaining fatal boot path (config validation, store health @@ -1193,7 +1216,7 @@ export async function runDaemonInner( } catch (err) { log( `[STORE] failed to start managed Oxigraph server: ${(err as Error).message}\n` + - `Fix the cause, or switch \`store.backend\` to oxigraph-worker (embedded) or ` + + `Fix the cause, or switch \`store.backend\` to ` + `sparql-http (operator-managed endpoint) in ~/.dkg/config.json.`, ); process.exit(1); @@ -1211,22 +1234,13 @@ export async function runDaemonInner( // For the directory-backed blob/snapshot stores we use the managed // defaults (the rewritten sparql-http backend has no `options.path` to // infer a directory from, unlike the local Oxigraph backend). - const runtimeStore = managed?.storeConfig ?? config.store; - const runtimeLargeLiteralStorage = - managed?.largeLiteralStorage ?? config.largeLiteralStorage; - const runtimeSnapshotStorage = - managed?.sharedMemoryPublicSnapshotStorage ?? config.sharedMemoryPublicSnapshotStorage; - // Config view used only for the boot-time store validation/health steps - // below: same as `config` but with the runtime store/blob/snapshot values - // swapped in, so a managed config validates against what actually runs. - const runtimeStoreConfig: DkgConfig = managed - ? { - ...config, - store: runtimeStore, - largeLiteralStorage: runtimeLargeLiteralStorage, - sharedMemoryPublicSnapshotStorage: runtimeSnapshotStorage, - } - : config; + const storeRuntime = resolveDaemonStoreRuntime(storeBoot, managed); + const { + runtimeStore, + runtimeConfig: runtimeStoreConfig, + runtimeLargeLiteralStorage, + runtimeSnapshotStorage, + } = storeRuntime; // Refuse to start on invalid external-backend config (missing URL, // missing blob/snapshot directory). This fires before the health @@ -1979,7 +1993,7 @@ export async function runDaemonInner( try { const runtime = await startPublisherRuntimeIfEnabled({ dataDir: dkgDir(), - config, + config: runtimeStoreConfig, store: agent.store, keypair: agent.wallet.keypair, chainBase: publisherChainBase, @@ -3360,7 +3374,7 @@ export async function runDaemonInner( agent, publisherControl, publisherRuntime, - config, + storeRuntime, startedAt, dashDb, opWallets, diff --git a/packages/cli/src/daemon/oxigraph-managed.ts b/packages/cli/src/daemon/oxigraph-managed.ts index 1dd6cd6b5f..948505a601 100644 --- a/packages/cli/src/daemon/oxigraph-managed.ts +++ b/packages/cli/src/daemon/oxigraph-managed.ts @@ -23,6 +23,7 @@ * matching the Blazegraph-Docker provisioner's contract. */ import { join } from 'node:path'; +import { MANAGED_DAEMON_STORE_BACKEND } from '../store-backends.js'; import { ensureOxigraphBinary } from './oxigraph-binary.js'; import { startOxigraphServer, @@ -35,7 +36,7 @@ import { } from './oxigraph-launch-strategy.js'; /** Config value that opts a node into the daemon-managed local server. */ -export const MANAGED_OXIGRAPH_BACKEND = 'oxigraph-server'; +export { MANAGED_DAEMON_STORE_BACKEND as MANAGED_OXIGRAPH_BACKEND }; /** Default loopback bind port. Override via `store.options.port`. */ export const DEFAULT_OXIGRAPH_PORT = 7878; @@ -144,7 +145,7 @@ export function planManagedOxigraph( config: ConfigLike, dataDir: string, ): ManagedOxigraphPlan | null { - if (config.store?.backend !== MANAGED_OXIGRAPH_BACKEND) return null; + if (config.store?.backend !== MANAGED_DAEMON_STORE_BACKEND) return null; const options = config.store.options ?? {}; const port = resolveManagedOxigraphPort(options); diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index 8b37447315..c29268c4b4 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -23,6 +23,7 @@ import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; import type { CatchupTracker } from '../types.js'; import type { RoutePlugin } from '../plugin-api.js'; import type { AdmissionStatsView } from '../http-utils.js'; +import type { StoreRuntimeContext } from '../store-runtime.js'; export type MemoryGraphLayer = 'wm' | 'swm' | 'vm'; @@ -54,7 +55,29 @@ export interface NotificationSseEvent { type: string; } -export interface RequestContext { +/** + * Store views exposed to routes. The operator config is intentionally the only + * config object in this shape, so a direct route harness cannot provide a + * second, contradictory operator config through a nested store context. + */ +export interface RequestStoreContext { + /** Operator config exactly as loaded from disk / CLI. */ + config: DkgConfig; + /** Daemon-facing backend after defaults and acknowledged migrations. */ + effectiveStore: StoreRuntimeContext['effectiveStore']; + /** Constructible live adapter config after managed-store materialization. */ + runtimeStore: StoreRuntimeContext['runtimeStore']; +} + +export function createRequestStoreContext(storeRuntime: StoreRuntimeContext): RequestStoreContext { + return { + config: storeRuntime.operatorConfig, + effectiveStore: storeRuntime.effectiveStore, + runtimeStore: storeRuntime.runtimeStore, + }; +} + +export interface RequestContext extends RequestStoreContext { req: IncomingMessage; res: ServerResponse; agent: DKGAgent; @@ -62,7 +85,6 @@ export interface RequestContext { publisherRuntime: PublisherRuntime | null; /** Lifecycle-owned publisher state; optional for direct route embeddings/tests. */ publisherAvailability?: AsyncPublisherAvailability; - config: DkgConfig; startedAt: number; dashDb: DashboardDB; opWallets: OpWalletsConfig; diff --git a/packages/cli/src/daemon/routes/status.ts b/packages/cli/src/daemon/routes/status.ts index 1ddf8eade3..aa052f979e 100644 --- a/packages/cli/src/daemon/routes/status.ts +++ b/packages/cli/src/daemon/routes/status.ts @@ -57,8 +57,10 @@ const execAsync = promisify(exec); const execFileAsync = promisify(execFile); import { enrichEvmError, MockChainAdapter, resolveRpcUrls, getRpcFailoverStats } from '@origintrail-official/dkg-chain'; import { DKGAgent, loadOpWallets } from '@origintrail-official/dkg-agent'; -import { isExternalBackend } from '@origintrail-official/dkg-storage'; -import { resolveManagedOxigraphPort } from '../oxigraph-managed.js'; +import { + isExternalStoreBackend as isExternalBackend, + isManagedLocalBackend, +} from '../../store-backends.js'; import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri } from '@origintrail-official/dkg-core'; import { findReservedSubjectPrefix, isSkolemizedUri } from '@origintrail-official/dkg-publisher'; import { @@ -420,6 +422,10 @@ export function invalidateExternalStoreQuadsCache(): void { storeQuadsInflight = null; } +export function storeBackendHasStatusHealth(backend: string | undefined): boolean { + return isExternalBackend(backend) || isManagedLocalBackend(backend); +} + async function getCachedExternalStoreQuads( agent: DKGAgent, now: number, @@ -498,6 +504,8 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { agent, publisherControl, config, + effectiveStore, + runtimeStore, startedAt, dashDb, opWallets, @@ -651,6 +659,7 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { // sentinels when build-info.json is absent (monorepo / dev), // so consumers can branch reliably. const buildInfo = loadBuildInfo(); + const runtimeStoreOptions = (runtimeStore.options ?? {}) as Record; return jsonResponse(res, 200, { name: config.name, version: nodeVersion, @@ -666,36 +675,25 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { networkConfig: resolveNetworkConfigName(config), networkId, networkName: network?.networkName ?? null, - storeBackend: config.store?.backend ?? "oxigraph-worker", + storeBackend: effectiveStore.backend, // External backend visibility (RFC 120 / plan PR 1 item 3). For // local backends both fields stay null so the response shape is // stable across deployments. - storeUrl: isExternalBackend(config.store?.backend) + storeUrl: isExternalBackend(runtimeStore.backend) ? (() => { - const opts = (config.store?.options ?? {}) as Record; - const url = typeof opts.url === 'string' ? opts.url - : typeof opts.queryEndpoint === 'string' ? opts.queryEndpoint + const url = typeof runtimeStoreOptions.url === 'string' ? runtimeStoreOptions.url + : typeof runtimeStoreOptions.queryEndpoint === 'string' ? runtimeStoreOptions.queryEndpoint : null; return url; })() - : config.store?.backend === 'oxigraph-server' - // Managed local server: report its loopback endpoint so `dkg status` - // renders the external-store health path (storeQuads/unreachable) - // instead of printing it like a quad-less local store. - ? (() => { - const opts = (config.store?.options ?? {}) as Record; - const port = resolveManagedOxigraphPort(opts); - return `http://127.0.0.1:${port}/query`; - })() - : null, - // A managed `oxigraph-server` keeps `config.store.backend` as - // "oxigraph-server" (so it persists/labels correctly), but its quad - // count is still worth surfacing — it's the only store-health signal - // for that backend (getStoreBytes is null, there's no store.nq), and a - // failed query here is how operators see the managed server is down - // (e.g. after a failed revive) instead of it always looking healthy. + : null, + // `storeBackend` describes effective daemon policy while URL and health + // come from the live constructible adapter. This matters for both the + // implicit managed default and acknowledged oxigraph-worker cutovers: + // operator config may be absent/retired, effective is oxigraph-server, + // and runtime is its materialized loopback sparql-http adapter. storeQuads: - isExternalBackend(config.store?.backend) || config.store?.backend === 'oxigraph-server' + (storeBackendHasStatusHealth(effectiveStore.backend) || isExternalBackend(runtimeStore.backend)) ? await getCachedExternalStoreQuads(agent, Date.now()) : null, uptimeMs: Date.now() - startedAt, diff --git a/packages/cli/src/daemon/store-runtime.ts b/packages/cli/src/daemon/store-runtime.ts new file mode 100644 index 0000000000..6f43d6dc15 --- /dev/null +++ b/packages/cli/src/daemon/store-runtime.ts @@ -0,0 +1,174 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { + DEFAULT_DAEMON_STORE_BACKEND, + isManagedLocalBackend, + isRetiredStoreBackend, + requireStorageAdapterBackend, + type StorageAdapterBackend, +} from '../store-backends.js'; +import type { DkgConfig } from '../config.js'; +import { readPersistedStoreBackend } from './daemon-state.js'; +import type { ManagedOxigraphResult } from './oxigraph-managed.js'; + +type StoreConfig = NonNullable; +export type RuntimeStoreConfig = Omit & { + backend: StorageAdapterBackend; +}; + +interface DaemonStoreOperatorContext { + /** Operator-facing config exactly as loaded from disk / CLI. */ + operatorConfig: DkgConfig; +} + +export interface InvalidDaemonStoreConfig extends DaemonStoreOperatorContext { + kind: 'invalid-config'; +} + +export interface BlockedLegacyStoreCutover extends DaemonStoreOperatorContext { + kind: 'blocked-legacy-cutover'; + message: string; +} + +export interface DaemonStoreBootPlan extends DaemonStoreOperatorContext { + kind: 'bootable'; + /** Config with the implicit daemon default materialized for boot steps. */ + effectiveConfig: DkgConfig; + /** Store backend used for backend-switch detection and managed startup. */ + effectiveStore: StoreConfig; + /** Non-fatal startup notice, e.g. acknowledged legacy default cutover. */ + notice?: string; +} + +export type DaemonStoreBootDecision = + | InvalidDaemonStoreConfig + | BlockedLegacyStoreCutover + | DaemonStoreBootPlan; + +export interface DaemonStoreRuntimePlan extends DaemonStoreBootPlan { + /** Store config consumed by validation, health probes, wipe, and the agent. */ + runtimeStore: RuntimeStoreConfig; + /** Config view with runtime store/blob/snapshot values swapped in. */ + runtimeConfig: DkgConfig; + runtimeLargeLiteralStorage: DkgConfig['largeLiteralStorage']; + runtimeSnapshotStorage: DkgConfig['sharedMemoryPublicSnapshotStorage']; +} + +/** Explicit store views threaded into request routing after startup. */ +export interface StoreRuntimeContext { + /** Persisted/operator intent. Routes that save config must use this view. */ + operatorConfig: DkgConfig; + /** Daemon-facing backend after defaults and acknowledged migrations. */ + effectiveStore: StoreConfig; + /** Constructible live adapter config after managed-store materialization. */ + runtimeStore: RuntimeStoreConfig; +} + +export function resolveEffectiveDaemonStore(config: Pick): StoreConfig { + return config.store ?? { backend: DEFAULT_DAEMON_STORE_BACKEND, options: {} }; +} + +export function resolveDaemonStoreBootPlan(opts: { + config: DkgConfig; + dataDir: string; + acceptStoreReset: boolean; +}): DaemonStoreBootDecision { + const { config, dataDir, acceptStoreReset } = opts; + const legacyStorePath = join(dataDir, 'store.nq'); + const legacyStoreExists = existsSync(legacyStorePath); + const retiredStoreConfigured = isRetiredStoreBackend(config.store?.backend); + const configuredForManagedServer = !config.store || isManagedLocalBackend(config.store.backend); + const previousBackend = readPersistedStoreBackend(dataDir); + const legacyCutoverAlreadyRecorded = isManagedLocalBackend(previousBackend); + const legacyCutoverRequired = legacyStoreExists + && (configuredForManagedServer || retiredStoreConfigured) + && !legacyCutoverAlreadyRecorded; + const migrateAcknowledgedRetiredStore = retiredStoreConfigured + && legacyCutoverRequired + && acceptStoreReset; + const effectiveStore = migrateAcknowledgedRetiredStore + ? resolveEffectiveDaemonStore({}) + : resolveEffectiveDaemonStore(config); + const effectiveConfig = config.store && !migrateAcknowledgedRetiredStore + ? config + : { ...config, store: effectiveStore }; + + // A retired worker config with no legacy data has no migration decision to + // make. Keep that state separate so callers cannot accidentally continue to + // managed startup with an invalid operator config. + if (retiredStoreConfigured && !legacyCutoverRequired) { + return { kind: 'invalid-config', operatorConfig: config }; + } + + if (legacyCutoverRequired && !acceptStoreReset) { + const legacySource = retiredStoreConfigured + ? `${config.store?.backend} backend` + : config.store + ? 'worker-backed store' + : 'implicit worker default'; + return { + kind: 'blocked-legacy-cutover', + operatorConfig: config, + message: + `[STORE] oxigraph-worker support has been removed, but this node has a legacy ` + + `store.nq from the old ${legacySource}.\n` + + `Set store.backend to "oxigraph-server" (or an external SPARQL backend) and ` + + `restart with DKG_ACCEPT_STORE_RESET=1 to acknowledge the fresh-store cutover. ` + + `The legacy store.nq file is left untouched for manual backup or migration.`, + }; + } + + let notice: string | undefined; + if (legacyCutoverRequired && acceptStoreReset) { + notice = retiredStoreConfigured + ? `[STORE] explicit ${config.store?.backend} is retired; using oxigraph-server after reset acknowledgement. Legacy store.nq is left untouched.` + : '[STORE] using oxigraph-server after reset acknowledgement. Legacy store.nq is left untouched.'; + } else if (!config.store && acceptStoreReset) { + notice = + '[STORE] no store block found; using oxigraph-server. Legacy store.nq, if present, is left untouched.'; + } + + return { + kind: 'bootable', + operatorConfig: config, + effectiveConfig, + effectiveStore, + ...(notice ? { notice } : {}), + }; +} + +export function resolveDaemonStoreRuntime( + bootPlan: DaemonStoreBootPlan, + managed: ManagedOxigraphResult | null, +): DaemonStoreRuntimePlan { + if (isManagedLocalBackend(bootPlan.effectiveStore.backend) && !managed) { + throw new Error( + `Managed daemon store "${bootPlan.effectiveStore.backend}" was not materialized to a storage adapter`, + ); + } + const candidateStore = managed?.storeConfig ?? bootPlan.effectiveStore; + const runtimeStore: RuntimeStoreConfig = { + ...candidateStore, + backend: requireStorageAdapterBackend(candidateStore.backend), + }; + const runtimeLargeLiteralStorage = + managed?.largeLiteralStorage ?? bootPlan.operatorConfig.largeLiteralStorage; + const runtimeSnapshotStorage = + managed?.sharedMemoryPublicSnapshotStorage ?? bootPlan.operatorConfig.sharedMemoryPublicSnapshotStorage; + const runtimeConfig: DkgConfig = managed + ? { + ...bootPlan.effectiveConfig, + store: runtimeStore, + largeLiteralStorage: runtimeLargeLiteralStorage, + sharedMemoryPublicSnapshotStorage: runtimeSnapshotStorage, + } + : bootPlan.effectiveConfig; + + return { + ...bootPlan, + runtimeStore, + runtimeConfig, + runtimeLargeLiteralStorage, + runtimeSnapshotStorage, + }; +} diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index adac992a45..0d07d43c27 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -6,6 +6,7 @@ import { ACKCollector, AsyncLiftRunner, DKGPublisher, FileWorkspacePublicSnapsho import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { loadNetworkConfig, resolveReadyChainConfig, type DkgConfig } from './config.js'; import { loadPublisherWallets } from './publisher-wallets.js'; +import { isManagedLocalBackend, isRetiredStoreBackend } from './store-backends.js'; export type { ACKTransportFactory } from '@origintrail-official/dkg-publisher'; @@ -588,6 +589,18 @@ function createChainRecoveryResolver( async function createPublisherStore(dataDir: string, config: DkgConfig): Promise { if (config.store) { + if (isManagedLocalBackend(config.store.backend)) { + throw new Error( + `Publisher commands for daemon-managed store "${config.store.backend}" require a running DKG daemon. ` + + 'Start the daemon and retry; daemon-down direct inspection cannot safely materialize the managed store.', + ); + } + if (isRetiredStoreBackend(config.store.backend)) { + throw new Error( + `Publisher commands cannot open retired store backend "${config.store.backend}" directly. ` + + 'Start the daemon to complete the acknowledged store migration, then retry.', + ); + } const storeConfig = config.store as any; return await createTripleStore({ ...storeConfig, @@ -606,7 +619,7 @@ async function createPublisherStore(dataDir: string, config: DkgConfig): Promise } return await createTripleStore({ - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(dataDir, 'store.nq') }, largeLiteralStorage: defaultLargeLiteralStorage(dataDir, config), }); @@ -633,7 +646,6 @@ export function createPublicSnapshotStore( function isLocalOxigraphStoreConfig(storeConfig: { backend?: unknown }): boolean { return storeConfig.backend === 'oxigraph' - || storeConfig.backend === 'oxigraph-worker' || storeConfig.backend === 'oxigraph-persistent'; } diff --git a/packages/cli/src/store-backends.ts b/packages/cli/src/store-backends.ts new file mode 100644 index 0000000000..87b78c8305 --- /dev/null +++ b/packages/cli/src/store-backends.ts @@ -0,0 +1,226 @@ +import { + STORAGE_ADAPTERS, + isExternalBackend, + isStorageAdapterBackend, + type StorageAdapterBackend, +} from '@origintrail-official/dkg-storage'; + +/** + * Operator-facing daemon policy composed on top of storage-owned adapter facts. + * + * This is the sole owner of daemon defaults, retired config names, migration + * classification, menu visibility, and labels. Adapter endpoint/path metadata + * is spread from `STORAGE_ADAPTERS` rather than duplicated here. + */ +export const STORE_BACKENDS = { + 'oxigraph-server': { + kind: 'managed-local', + adapter: false, + retired: false, + default: true, + wizard: true, + storeFlag: true, + label: 'oxigraph-server (managed local server — recommended)', + }, + oxigraph: { + ...STORAGE_ADAPTERS.oxigraph, + adapter: true, + retired: false, + default: false, + wizard: true, + storeFlag: true, + label: 'oxigraph (embedded in-memory store — development only)', + }, + 'oxigraph-persistent': { + ...STORAGE_ADAPTERS['oxigraph-persistent'], + adapter: true, + retired: false, + default: false, + wizard: false, + storeFlag: false, + }, + blazegraph: { + ...STORAGE_ADAPTERS.blazegraph, + adapter: true, + retired: false, + default: false, + wizard: true, + storeFlag: true, + label: 'blazegraph (external SPARQL endpoint)', + }, + 'sparql-http': { + ...STORAGE_ADAPTERS['sparql-http'], + adapter: true, + retired: false, + default: false, + wizard: false, + storeFlag: true, + }, + 'oxigraph-worker': { + kind: 'retired', + adapter: false, + retired: true, + default: false, + wizard: false, + storeFlag: false, + }, +} as const; + +export type StoreBackend = keyof typeof STORE_BACKENDS; +export type StoreBackendPolicy = (typeof STORE_BACKENDS)[StoreBackend]; +export type StoreBackendKind = StoreBackendPolicy['kind']; +export type StoreBackendOfKind = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { kind: Kind } + ? Backend + : never; +}[StoreBackend]; +export type ConfigStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { retired: false } + ? Backend + : never; +}[StoreBackend]; +export type RetiredStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { retired: true } + ? Backend + : never; +}[StoreBackend]; +export type DefaultStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { default: true } + ? Backend + : never; +}[StoreBackend]; +export type WizardStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { wizard: true } + ? Backend + : never; +}[StoreBackend]; +export type StoreFlagBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { storeFlag: true } + ? Backend + : never; +}[StoreBackend]; +export type ExternalStoreBackend = Extract, StorageAdapterBackend>; +export type LocalStoreBackend = Extract, StorageAdapterBackend>; +export type ManagedLocalStoreBackend = StoreBackendOfKind<'managed-local'>; + +export function storeBackendNames(): StoreBackend[] { + return Object.keys(STORE_BACKENDS) as StoreBackend[]; +} + +function requireSingleBackend( + backends: readonly Backend[], + description: string, +): Backend { + if (backends.length !== 1) { + throw new Error(`Expected exactly one ${description} store backend, found ${backends.length}`); + } + return backends[0]; +} + +export const DEFAULT_DAEMON_STORE_BACKEND: DefaultStoreBackend = requireSingleBackend( + storeBackendNames().filter( + (backend): backend is DefaultStoreBackend => STORE_BACKENDS[backend].default, + ), + 'default', +); + +export const MANAGED_DAEMON_STORE_BACKEND: ManagedLocalStoreBackend = requireSingleBackend( + storeBackendNames().filter( + (backend): backend is ManagedLocalStoreBackend => STORE_BACKENDS[backend].kind === 'managed-local', + ), + 'managed-local', +); + +export const DEFAULT_STORE_BACKEND = DEFAULT_DAEMON_STORE_BACKEND; +export const MANAGED_LOCAL_STORE_BACKEND = MANAGED_DAEMON_STORE_BACKEND; + +export function configBackendNames(): ConfigStoreBackend[] { + return storeBackendNames().filter( + (backend): backend is ConfigStoreBackend => !STORE_BACKENDS[backend].retired, + ); +} + +export function retiredBackendNames(): RetiredStoreBackend[] { + return storeBackendNames().filter( + (backend): backend is RetiredStoreBackend => STORE_BACKENDS[backend].retired, + ); +} + +export function configBackendList(separator = ', '): string { + return configBackendNames().join(separator); +} + +export function wizardBackendChoices(): WizardStoreBackend[] { + return storeBackendNames().filter( + (backend): backend is WizardStoreBackend => + !STORE_BACKENDS[backend].retired && STORE_BACKENDS[backend].wizard, + ); +} + +export function storeFlagBackendNames(): StoreFlagBackend[] { + return storeBackendNames().filter( + (backend): backend is StoreFlagBackend => + !STORE_BACKENDS[backend].retired && STORE_BACKENDS[backend].storeFlag, + ); +} + +export function storeFlagBackendList(separator = ', '): string { + return storeFlagBackendNames().join(separator); +} + +export function isKnownStoreBackend( + backend: string | undefined | null, +): backend is StoreBackend { + return backend != null && Object.prototype.hasOwnProperty.call(STORE_BACKENDS, backend); +} + +export function getStoreBackendPolicy( + backend: string | undefined | null, +): StoreBackendPolicy | undefined { + return isKnownStoreBackend(backend) ? STORE_BACKENDS[backend] : undefined; +} + +export function isConfigStoreBackend( + backend: string | undefined | null, +): backend is ConfigStoreBackend { + return isKnownStoreBackend(backend) && !STORE_BACKENDS[backend].retired; +} + +export function isStoreFlagBackend( + backend: string | undefined | null, +): backend is StoreFlagBackend { + return isKnownStoreBackend(backend) + && !STORE_BACKENDS[backend].retired + && STORE_BACKENDS[backend].storeFlag; +} + +export function isRetiredStoreBackend( + backend: string | undefined | null, +): backend is RetiredStoreBackend { + return isKnownStoreBackend(backend) && STORE_BACKENDS[backend].retired; +} + +export function isManagedLocalBackend( + backend: string | undefined | null, +): backend is ManagedLocalStoreBackend { + return isKnownStoreBackend(backend) && STORE_BACKENDS[backend].kind === 'managed-local'; +} + +export function isExternalStoreBackend( + backend: string | undefined | null, +): backend is ExternalStoreBackend { + return isExternalBackend(backend); +} + +/** Cross from daemon runtime config into the storage factory's adapter type. */ +export function requireStorageAdapterBackend(backend: string): StorageAdapterBackend { + if (!isStorageAdapterBackend(backend)) { + throw new Error( + `Daemon runtime store backend "${backend}" is not a constructible storage adapter`, + ); + } + return backend; +} + +export { isStorageAdapterBackend }; +export type { StorageAdapterBackend }; diff --git a/packages/cli/src/store-wizard.ts b/packages/cli/src/store-wizard.ts index 4eac50b144..768294bc71 100644 --- a/packages/cli/src/store-wizard.ts +++ b/packages/cli/src/store-wizard.ts @@ -26,6 +26,21 @@ import { type ProvisionBlazegraphDockerOptions, type ProvisionBlazegraphDockerResult, } from './daemon/blazegraph-docker.js'; +import { + DEFAULT_STORE_BACKEND, + MANAGED_LOCAL_STORE_BACKEND, + STORE_BACKENDS, + configBackendList, + isConfigStoreBackend, + isRetiredStoreBackend, + isStoreFlagBackend, + storeFlagBackendList, + wizardBackendChoices, + type ConfigStoreBackend, + type ExternalStoreBackend, + type StoreFlagBackend, + type WizardStoreBackend, +} from './store-backends.js'; export interface PromptStoreBackendOptions { /** `ask` callback that closes over a shared readline interface. */ @@ -67,9 +82,9 @@ export interface PromptStoreBackendOptions { export interface PromptStoreBackendResult { /** - * Persisted store block. `null` means "use the local default" (caller - * should omit the field or set it to undefined; saveConfig drops - * undefined keys). + * Persisted store block. `null` means "leave the store block omitted"; + * daemon boot treats an omitted store block as the managed `oxigraph-server` + * default. * * `managedByDkg: true` is set by the Docker provisioner branch only; * manual URLs always get `managedByDkg: false`. The chain-reset-wipe @@ -80,15 +95,8 @@ export interface PromptStoreBackendResult { storeBlock: ExternalStoreBlock | LocalStoreBlock | null; } -// An explicit embedded/local store block carried through verbatim on an -// Enter-through re-init. `null` still means "no block — runtime default", but -// when a node already pinned a local backend with custom `options` (e.g. the -// `options.path` the oxigraph-worker adapter reads for its persistence file), -// returning that block instead of `null` keeps re-init idempotent: cli init -// writes `store: storeBlock ?? undefined`, so a `null` would clear the block -// and relocate the store on the next boot. export type LocalStoreBlock = { - backend: 'oxigraph' | 'oxigraph-worker' | 'oxigraph-persistent'; + backend: 'oxigraph' | 'oxigraph-persistent'; options?: Record; }; @@ -110,7 +118,7 @@ export type ExternalStoreBlock = // endpoints at boot. Operator-set overrides (`port`/`location`/`cacheDir`) // that planManagedOxigraph reads at boot are carried through unchanged. | { - backend: 'oxigraph-server'; + backend: typeof MANAGED_LOCAL_STORE_BACKEND; options: Record; }; @@ -136,6 +144,39 @@ function externalStoreBlock( }; } +function isSupportedExistingBackend(backend: string | undefined): backend is ConfigStoreBackend { + return isConfigStoreBackend(backend); +} + +function retiredBackendError(source: string, backend: string): Error { + const choices = source === '--store' ? storeFlagBackendList() : configBackendList(); + return new Error( + `${source} "${backend}" is no longer supported. ` + + `Use one of: ${choices}.`, + ); +} + +function unknownBackendError(source: 'prompt' | '--store', backend: string): Error { + if (source === '--store') { + return new Error(`--store must be one of: ${storeFlagBackendList()} (got "${backend}")`); + } + return new Error(`Unknown store backend "${backend}". Expected one of: ${configBackendList()}.`); +} + +function parseBackendAnswer( + input: string, + defaultBackend: string, + choices: readonly WizardStoreBackend[], +): string { + return /^\d+$/.test(input) + ? (choices[parseInt(input, 10) - 1] ?? defaultBackend) + : input.toLowerCase(); +} + +function hasStorePath(store: PromptStoreBackendOptions['existingStore'] | undefined): boolean { + return typeof store?.options?.path === 'string' && store.options.path.trim().length > 0; +} + export async function promptStoreBackend( opts: PromptStoreBackendOptions, ): Promise { @@ -156,35 +197,24 @@ export async function promptStoreBackend( : undefined; // `oxigraph-server` (daemon-managed local RocksDB server) is the default - // local backend: it gives MVCC concurrent reads + incremental persistence, - // whereas `oxigraph` (the embedded in-process worker) rewrites the whole - // N-Quads dump on every flush. The in-process worker stays available as a - // minimal-footprint / single-reader option. NOTE: this only changes what a - // *fresh / block-less* `dkg init` writes — the runtime fallback for configs - // with no `store` block stays `oxigraph-worker`, so the existing fleet keeps - // booting unchanged on auto-update (only an explicit re-init flips a node, - // and the daemon's STORE-SWITCH guard makes that an opt-in, not silent). - // Keep ANY explicit existing backend as the default answer — including the - // embedded `oxigraph` / `oxigraph-worker` / `oxigraph-persistent` variants. - // Keeping the *exact* variant (rather than normalising the worker variants - // onto the listed `oxigraph` choice) matters for distinguishing intent: an - // Enter-through resolves the default back to that exact backend (so the - // preserve branch below keeps the block + custom options), whereas explicitly - // picking option `2` ("oxigraph") resolves to a *different* answer and is - // treated as a real switch to the plain embedded worker. Only a truly absent - // store config (fresh install / block-less node) falls through to the new - // `oxigraph-server` default. - const defaultBackend = opts.flagBackend - ?? (existingBackend === 'blazegraph' || existingBackend === 'sparql-http' || existingBackend === 'oxigraph-server' - || existingBackend === 'oxigraph' || existingBackend === 'oxigraph-worker' || existingBackend === 'oxigraph-persistent' + // local backend: it gives MVCC concurrent reads + incremental persistence. + // The old `oxigraph-worker` fallback is retired; configs that still name it + // are not preserved on Enter-through. + if (isRetiredStoreBackend(existingBackend)) { + log(` Existing store.backend "${existingBackend}" is retired; defaulting to oxigraph-server.`); + } + const flagBackend = opts.flagBackend?.trim().toLowerCase(); + if (flagBackend && !isStoreFlagBackend(flagBackend)) { + if (isRetiredStoreBackend(flagBackend)) { + throw retiredBackendError('--store', flagBackend); + } + throw unknownBackendError('--store', flagBackend); + } + const defaultBackend = flagBackend + ?? (isSupportedExistingBackend(existingBackend) ? existingBackend - : 'oxigraph-server'); - const backendChoices = ['oxigraph-server', 'oxigraph', 'blazegraph'] as const; - const backendLabels: Record = { - 'oxigraph-server': 'oxigraph-server (managed local server — recommended)', - 'oxigraph': 'oxigraph (embedded in-process worker)', - 'blazegraph': 'blazegraph (external SPARQL endpoint)', - }; + : DEFAULT_STORE_BACKEND); + const backendChoices = wizardBackendChoices(); // `sparql-http` is intentionally not listed (advanced bring-your-own-server // option) but is still accepted when typed or inherited from an existing // config / `--store` flag. Resolve the default *answer* by name for unlisted @@ -198,7 +228,7 @@ export async function promptStoreBackend( log(' Triple store backend:'); for (let i = 0; i < backendChoices.length; i++) { const choice = backendChoices[i]; - log(` ${i + 1}) ${backendLabels[choice] ?? choice}`); + log(` ${i + 1}) ${STORE_BACKENDS[choice].label}`); } // When the inherited/flagged backend isn't one of the numbered choices // (e.g. `sparql-http`), spell out that pressing Enter keeps it and that a @@ -216,49 +246,52 @@ export async function promptStoreBackend( // out-of-range number (typo like "4") falls back to `defaultBackend` — // i.e. the recommended option shown to the operator — rather than a // hard-coded `oxigraph`, so a fat-fingered digit on a fresh install no - // longer silently downgrades the node to the embedded worker. - const backendAnswer = /^\d+$/.test(backendInput) - ? (backendChoices[parseInt(backendInput, 10) - 1] ?? defaultBackend) - : backendInput.toLowerCase(); + // longer silently downgrades the node to an embedded backend. + const backendAnswer = parseBackendAnswer(backendInput, defaultBackend, backendChoices); + + if (isRetiredStoreBackend(backendAnswer)) { + throw retiredBackendError('store backend', backendAnswer); + } + if (!isSupportedExistingBackend(backendAnswer)) { + throw unknownBackendError('prompt', backendAnswer); + } + const backendPolicy = STORE_BACKENDS[backendAnswer]; // `oxigraph-server` (daemon-managed local server) is the default numbered // choice and is also accepted by name. No URL prompt or probe: the endpoint // doesn't exist until the daemon spawns it at boot. - if (backendAnswer === 'oxigraph-server') { + if (backendPolicy.kind === 'managed-local') { log(' Using a daemon-managed local Oxigraph server (started on first daemon boot).'); // Preserve existing managed-server overrides (port/location/cacheDir) on an // Enter-through: `dkg init` persists this block, so returning empty options // would silently reset a custom port/RocksDB path on the next boot — the // same hazard applyStoreFlagsToConfig guards against on the `--store` path. const prevOptions = - existingBackend === 'oxigraph-server' && opts.existingStore?.options + existingBackend === MANAGED_LOCAL_STORE_BACKEND && opts.existingStore?.options ? opts.existingStore.options : {}; - return { storeBlock: { backend: 'oxigraph-server', options: prevOptions } }; + return { storeBlock: { backend: MANAGED_LOCAL_STORE_BACKEND, options: prevOptions } }; } - if (backendAnswer !== 'blazegraph' && backendAnswer !== 'sparql-http') { - // Embedded in-process worker. Preserve an existing explicit local store - // block verbatim ONLY when the operator kept that same backend — i.e. the - // resolved answer equals the existing backend (an Enter-through, which - // resolves the default back to the exact existing variant). That keeps a - // re-init idempotent and never drops custom `options` (e.g. the worker's - // `options.path`) or relocates the store. An *explicit* switch — picking - // option `2`/"oxigraph" on a node currently using `oxigraph-worker` / - // `oxigraph-persistent`, a switch from an external/server backend, or a - // fresh / block-less init — resolves to a different answer and falls - // through to `null`, so `dkg init` clears the old block as intended. - if ( - (backendAnswer === 'oxigraph' || - backendAnswer === 'oxigraph-worker' || - backendAnswer === 'oxigraph-persistent') && - backendAnswer === existingBackend - ) { - return { storeBlock: { backend: existingBackend, options: opts.existingStore?.options } }; + if (backendPolicy.kind === 'local') { + const localBackend = backendAnswer as LocalStoreBlock['backend']; + if (STORE_BACKENDS[localBackend].requiresExistingPath && !hasStorePath(opts.existingStore)) { + throw new Error( + 'store backend "oxigraph-persistent" requires store.options.path; ' + + 'set it manually in config.json or use "oxigraph-server".', + ); } - return { storeBlock: null }; + const prevOptions = + localBackend === existingBackend && opts.existingStore?.options + ? opts.existingStore.options + : {}; + return { storeBlock: { backend: localBackend, options: prevOptions } }; + } + + if (backendPolicy.kind !== 'external') { + throw unknownBackendError('prompt', backendAnswer); } - const backend = backendAnswer as 'blazegraph' | 'sparql-http'; + const backend = backendAnswer as ExternalStoreBackend; // URL prompt loop: validate each attempt, surface the operator-facing // failure message, allow retry or abort. @@ -317,14 +350,13 @@ export async function promptStoreBackend( } const retry = (await opts.ask('Retry with a URL? (y/n)', 'y')).toLowerCase(); if (retry === 'n') { - log(' Aborting store setup; defaulting to local Oxigraph.'); + log(' Aborting store setup; using the oxigraph-server default.'); return { storeBlock: null }; } continue; } - const optionsForProbe = - backend === 'blazegraph' ? { url } : { queryEndpoint: url }; + const optionsForProbe = { [backendPolicy.queryEndpointOption]: url }; const health = await checkExternalStoreReachable({ storeConfig: { backend, options: optionsForProbe }, fetch: opts.fetch, @@ -352,7 +384,7 @@ export async function promptStoreBackend( 'y', )).toLowerCase(); if (retry === 'n') { - log(' Aborting store setup; defaulting to local Oxigraph.'); + log(' Aborting store setup; using the oxigraph-server default.'); return { storeBlock: null }; } } @@ -391,61 +423,64 @@ export async function applyStoreFlagsToConfig( opts: ApplyStoreFlagsOptions, ): Promise { const log = opts.log ?? console.log; - const backend = opts.storeFlag; + const backend = opts.storeFlag?.trim().toLowerCase(); if (!backend) return; const load = opts.loadConfig ?? loadConfig; const save = opts.saveConfig ?? saveConfig; - // Operators who pass `--store oxigraph` may be trying to FORCE local - // even though their existing config has a `store` block — honour - // that by clearing the block. - if ( - backend === 'oxigraph' || - backend === 'oxigraph-worker' || - backend === 'oxigraph-persistent' - ) { + if (isRetiredStoreBackend(backend)) { + throw retiredBackendError('--store', backend); + } + if (!isStoreFlagBackend(backend)) { + throw unknownBackendError('--store', backend); + } + const backendPolicy = STORE_BACKENDS[backend]; + + // Operators who pass `--store oxigraph` are explicitly opting into the + // embedded development store. Persist it because an omitted store block now + // means the managed `oxigraph-server` default. + if (backendPolicy.kind === 'local') { + const localBackend = backend as Extract; const existing = await load(); - if (existing.store) { - log(` Removing existing store block (--store ${backend} → local default).`); - const next = { ...existing }; - delete next.store; - await save(next); + await save({ ...existing, store: { backend: localBackend, options: {} } }); + if (localBackend === 'oxigraph') { + log(' Store configured: oxigraph (embedded development store).'); + } else { + log(` Store configured: ${localBackend}.`); } return; } // Daemon-managed local Oxigraph server: no URL to validate (the daemon // brings it up at boot). Write the block and return. - if (backend === 'oxigraph-server') { + if (backendPolicy.kind === 'managed-local') { const existing = await load(); // Preserve any existing managed-server overrides (port/location/cacheDir) // that planManagedOxigraph reads at boot — re-running setup with // `--store oxigraph-server` must not silently reset them to defaults. const prevOptions = - existing.store?.backend === 'oxigraph-server' && existing.store.options + existing.store?.backend === MANAGED_LOCAL_STORE_BACKEND && existing.store.options ? existing.store.options : {}; - await save({ ...existing, store: { backend: 'oxigraph-server', options: prevOptions } }); + await save({ ...existing, store: { backend: MANAGED_LOCAL_STORE_BACKEND, options: prevOptions } }); log(' Store configured: oxigraph-server (daemon-managed local server).'); return; } - if (backend !== 'blazegraph' && backend !== 'sparql-http') { - throw new Error( - `--store must be one of: oxigraph, blazegraph, sparql-http, oxigraph-server (got "${backend}")`, - ); + if (backendPolicy.kind !== 'external') { + throw unknownBackendError('--store', backend); } + const externalBackend = backend as ExternalStoreBackend; const url = opts.storeUrlFlag?.trim(); if (!url) { - throw new Error(`--store ${backend} requires --store-url `); + throw new Error(`--store ${externalBackend} requires --store-url `); } - const optionsForProbe = - backend === 'blazegraph' ? { url } : { queryEndpoint: url }; + const optionsForProbe = { [backendPolicy.queryEndpointOption]: url }; const health = await checkExternalStoreReachable({ - storeConfig: { backend, options: optionsForProbe }, + storeConfig: { backend: externalBackend, options: optionsForProbe }, fetch: opts.fetch, }); if (!health.ok) { @@ -455,8 +490,8 @@ export async function applyStoreFlagsToConfig( const existing = await load(); const next: DkgConfig = { ...existing, - store: externalStoreBlock(backend, url, false), + store: externalStoreBlock(externalBackend, url, false), }; await save(next); - log(` Store configured: ${backend} (${url}) — verified reachable.`); + log(` Store configured: ${externalBackend} (${url}) — verified reachable.`); } diff --git a/packages/cli/test/chain-reset-wipe.test.ts b/packages/cli/test/chain-reset-wipe.test.ts index 909ebce67b..6db6e56d4d 100644 --- a/packages/cli/test/chain-reset-wipe.test.ts +++ b/packages/cli/test/chain-reset-wipe.test.ts @@ -589,7 +589,7 @@ describe('chainResetWipe — external SPARQL wipe', () => { expect(result.failedFiles.some((f) => f.error.includes('ECONNREFUSED'))).toBe(true); }); - it('skips external wipe entirely for local backends (storeConfig present, backend oxigraph-worker)', async () => { + it('skips external wipe entirely for local backends (storeConfig present, backend oxigraph-persistent)', async () => { writeFileSync(join(dataDir, 'store.nq'), '

.'); const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); @@ -597,7 +597,7 @@ describe('chainResetWipe — external SPARQL wipe', () => { dataDir, currentMarker: NEW_MARKER, storeConfig: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', // Options that LOOK like an external URL: these must be ignored // because the backend itself is local. Otherwise an operator who // hand-tuned options would see surprise SPARQL requests. @@ -652,7 +652,7 @@ describe('detectBackendSwitch', () => { const logs: string[] = []; const result = detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-worker', + currentBackend: 'oxigraph-persistent', acceptStoreReset: false, log: (m) => logs.push(m), }); @@ -662,7 +662,7 @@ describe('detectBackendSwitch', () => { expect(result.aborted).toBe(false); const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); - expect(persisted.lastBackend).toBe('oxigraph-worker'); + expect(persisted.lastBackend).toBe('oxigraph-persistent'); // First-boot path is silent — no STORE-SWITCH warning header. expect(logs.find((l) => l.includes('STORE-SWITCH'))).toBeUndefined(); }); @@ -696,19 +696,19 @@ describe('detectBackendSwitch', () => { it('is a no-op when backend matches previous boot', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), ); const logs: string[] = []; const result = detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-worker', + currentBackend: 'oxigraph-persistent', acceptStoreReset: false, log: (m) => logs.push(m), }); expect(result.changed).toBe(false); - expect(result.previous).toBe('oxigraph-worker'); + expect(result.previous).toBe('oxigraph-persistent'); expect(result.aborted).toBe(false); expect(logs).toEqual([]); }); @@ -716,7 +716,7 @@ describe('detectBackendSwitch', () => { it('aborts boot on mismatch without acceptStoreReset, surfaces multi-line warning', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), ); const logs: string[] = []; @@ -728,25 +728,25 @@ describe('detectBackendSwitch', () => { }); expect(result.changed).toBe(true); - expect(result.previous).toBe('oxigraph-worker'); + expect(result.previous).toBe('oxigraph-persistent'); expect(result.aborted).toBe(true); const joined = logs.join('\n'); expect(joined).toMatch(/STORE-SWITCH/); - expect(joined).toMatch(/previous: oxigraph-worker/); + expect(joined).toMatch(/previous: oxigraph-persistent/); expect(joined).toMatch(/current:\s+blazegraph/); expect(joined).toMatch(/DKG_ACCEPT_STORE_RESET=1/); // State file MUST still record the old backend so a corrected // config (operator reverts the edit) sees a match on next boot. const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); - expect(persisted.lastBackend).toBe('oxigraph-worker'); + expect(persisted.lastBackend).toBe('oxigraph-persistent'); }); it('proceeds and updates state when acceptStoreReset=true', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), ); const logs: string[] = []; @@ -775,12 +775,12 @@ describe('detectBackendSwitch', () => { // Establish a baseline by running detectBackendSwitch first. detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-worker', + currentBackend: 'oxigraph-persistent', acceptStoreReset: false, }); expect( JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')).lastBackend, - ).toBe('oxigraph-worker'); + ).toBe('oxigraph-persistent'); // Now run a marker change — the wipe path persists chainResetMarker // and MUST preserve lastBackend. @@ -792,7 +792,7 @@ describe('detectBackendSwitch', () => { const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); expect(persisted.chainResetMarker).toBe(NEW_MARKER); - expect(persisted.lastBackend).toBe('oxigraph-worker'); + expect(persisted.lastBackend).toBe('oxigraph-persistent'); }); }); diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index c4f6a30152..3f2e3a10b4 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -102,7 +102,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: authEnabled }, store: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(home, 'store.nq') }, }, // Real EVM adapter against the shared Hardhat node (port 9548 per diff --git a/packages/cli/test/daemon-http-inflight-cap.test.ts b/packages/cli/test/daemon-http-inflight-cap.test.ts index e38b58c48f..c36f9b0749 100644 --- a/packages/cli/test/daemon-http-inflight-cap.test.ts +++ b/packages/cli/test/daemon-http-inflight-cap.test.ts @@ -1,4 +1,5 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { request as httpRequest } from 'node:http'; import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from './helpers/live-daemon.js'; /** @@ -11,13 +12,11 @@ import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from '. * http-admission-control.test.ts: it would fail if the limiter were never wired * into createServer, wired after an early return, or never released. * - * Saturation is created with a 50-request burst against cap=1 rather than a - * single held-open request. That is statistically deterministic — 50 concurrent - * requests cannot all serialize through one slot without overlap — and avoids a - * brittle blocking fixture (the daemon admits before the route reads the body, - * so an unfinished-body "hold" does not reliably pin the slot). The precise - * one-in/one-shed/release semantics are covered deterministically by the unit - * tests; here we prove the wiring end-to-end. + * Saturation is created with a real HTTP request whose JSON body is deliberately + * left unfinished. The test then polls the admission-exempt status route until + * it observes the occupied slot before sending competing requests. This proves + * the production wiring without relying on a fast request burst happening to + * overlap on a particular runner. */ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { let daemon: LiveDaemon | undefined; @@ -46,24 +45,76 @@ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { }); } + async function readAdmission( + d: LiveDaemon, + ): Promise<{ inFlight: number; max: number; rejectedTotal: number }> { + const res = await fetch(`${d.base}/api/status`, { headers: authHeaders(d) }); + expect(res.status).toBe(200); + const body = (await res.json()) as { + admission?: { inFlight: number; max: number; rejectedTotal: number }; + }; + expect(body.admission).toBeDefined(); + return body.admission!; + } + + async function holdQuerySlot(d: LiveDaemon): Promise<{ + release: () => void; + responseStatus: Promise; + }> { + let released = false; + let req: ReturnType; + const responseStatus = new Promise((resolve, reject) => { + req = httpRequest( + `${d.base}/api/query`, + { + method: 'POST', + headers: authHeaders(d), + }, + (res) => { + res.resume(); + res.once('end', () => resolve(res.statusCode ?? 0)); + }, + ); + req.once('error', reject); + req.write('{"sparql":"SELECT * WHERE { ?s ?p ?o } LIMIT 1","hold":"'); + }); + const release = () => { + if (released) return; + released = true; + req.end('released"}'); + }; + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if ((await readAdmission(d)).inFlight === 1) { + return { release, responseStatus }; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + release(); + throw new Error('Timed out waiting for the held query to occupy the admission slot'); + } + it('sheds concurrent over-capacity requests with 503 + Retry-After, then recovers', async () => { const d = daemon!; - const results = await Promise.all( - Array.from({ length: 50 }, () => - selectQuery(d) - .then((r) => ({ status: r.status, retryAfter: r.headers.get('retry-after') })) - .catch(() => ({ status: 0, retryAfter: null as string | null })), - ), - ); - const shed = results.filter((r) => r.status === 503); - const ok = results.filter((r) => r.status === 200); - - // Every result must be an EXPECTED status — never a network error (0) or an - // unexpected 4xx/5xx that would otherwise hide behind the >=1/>=1 counts. - expect(results.every((r) => r.status === 200 || r.status === 503)).toBe(true); - expect(ok.length).toBeGreaterThan(0); // at least one admitted - expect(shed.length).toBeGreaterThan(0); // cap enforced under concurrent load - expect(shed.every((r) => r.retryAfter === '1')).toBe(true); // Retry-After present on every 503 + const held = await holdQuerySlot(d); + try { + const results = await Promise.all( + Array.from({ length: 10 }, () => + selectQuery(d) + .then((r) => ({ status: r.status, retryAfter: r.headers.get('retry-after') })) + .catch(() => ({ status: 0, retryAfter: null as string | null })), + ), + ); + + expect(results.every((r) => r.status === 503)).toBe(true); + expect(results.every((r) => r.retryAfter === '1')).toBe(true); + + held.release(); + expect(await held.responseStatus).toBe(200); + } finally { + held.release(); + } // Slots are released after each handler completes → a fresh request succeeds. const recovered = await selectQuery(d); @@ -72,58 +123,54 @@ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { it('keeps the exempt liveness path (/api/status) answerable even while saturated', async () => { const d = daemon!; - // Saturate with non-exempt query work; capture the burst results so we can - // PROVE the daemon was actually over capacity (>=1 shed) while the status - // probes ran — otherwise "status stayed 200" would be vacuous. - const burst = Promise.all( - Array.from({ length: 40 }, () => - selectQuery(d).then((r) => r.status).catch(() => 0), - ), - ); - // ...while hammering the exempt status endpoint, which must always answer 200. - const statuses = await Promise.all( - Array.from({ length: 12 }, () => - fetch(`${d.base}/api/status`, { headers: authHeaders(d) }) - .then((r) => r.status) - .catch(() => 0), - ), - ); - const burstStatuses = await burst; - - expect(statuses.every((s) => s === 200)).toBe(true); // exempt path never shed - expect(burstStatuses.filter((s) => s === 503).length).toBeGreaterThan(0); // saturation really happened - expect(burstStatuses.every((s) => s === 200 || s === 503)).toBe(true); // no unexpected failures + const held = await holdQuerySlot(d); + try { + const [statuses, burstStatuses] = await Promise.all([ + Promise.all( + Array.from({ length: 12 }, () => + fetch(`${d.base}/api/status`, { headers: authHeaders(d) }) + .then((r) => r.status) + .catch(() => 0), + ), + ), + Promise.all( + Array.from({ length: 10 }, () => + selectQuery(d).then((r) => r.status).catch(() => 0), + ), + ), + ]); + + expect(statuses.every((s) => s === 200)).toBe(true); + expect(burstStatuses.every((s) => s === 503)).toBe(true); + } finally { + held.release(); + } + expect(await held.responseStatus).toBe(200); }, 60_000); it('surfaces admission stats on /api/status (effective cap + per-burst shed delta)', async () => { const d = daemon!; - // Read the surfaced admission block off the exempt status endpoint. - const readAdmission = async (): Promise<{ inFlight: number; max: number; rejectedTotal: number }> => { - const res = await fetch(`${d.base}/api/status`, { headers: authHeaders(d) }); - expect(res.status).toBe(200); - const body = (await res.json()) as { - admission?: { inFlight: number; max: number; rejectedTotal: number }; - }; - expect(body.admission).toBeDefined(); - return body.admission!; - }; - // Snapshot BEFORE this burst — earlier tests in this file already shed, so a // bare `rejectedTotal > 0` would pass without proving THIS burst moved the // counter (i.e. that the surfaced value still tracks live shedding). - const before = await readAdmission(); + const before = await readAdmission(d); expect(before.max).toBe(1); // the pinned effective cap is surfaced expect(typeof before.inFlight).toBe('number'); - // Saturate the non-exempt path so this burst provably sheds. - const burst = await Promise.all( - Array.from({ length: 50 }, () => selectQuery(d).then((r) => r.status).catch(() => 0)), - ); - expect(burst.filter((s) => s === 503).length).toBeGreaterThan(0); // this burst really shed + const held = await holdQuerySlot(d); + try { + const burst = await Promise.all( + Array.from({ length: 10 }, () => selectQuery(d).then((r) => r.status).catch(() => 0)), + ); + expect(burst.every((s) => s === 503)).toBe(true); + } finally { + held.release(); + } + expect(await held.responseStatus).toBe(200); // /api/status is admission-exempt, so reading it doesn't perturb the counter: // `after` MUST exceed `before` by the sheds we just caused. - const after = await readAdmission(); + const after = await readAdmission(d); expect(after.rejectedTotal).toBeGreaterThan(before.rejectedTotal); }, 60_000); }); diff --git a/packages/cli/test/daemon-startup-validation.test.ts b/packages/cli/test/daemon-startup-validation.test.ts index cef225b46b..cee7688fb6 100644 --- a/packages/cli/test/daemon-startup-validation.test.ts +++ b/packages/cli/test/daemon-startup-validation.test.ts @@ -1,5 +1,5 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { computeNetworkId } from '../../core/src/genesis.js'; @@ -9,6 +9,9 @@ const mocks = vi.hoisted(() => ({ agentCreate: vi.fn(), loadOpWallets: vi.fn(), loadNetworkConfig: vi.fn(), + checkExternalStoreReachable: vi.fn(), + checkOrSetStoreIdentity: vi.fn(), + startManagedOxigraph: vi.fn(), })); vi.mock('@origintrail-official/dkg-agent', async importOriginal => { @@ -29,6 +32,23 @@ vi.mock('../src/config.js', async importOriginal => { }; }); +vi.mock('../src/daemon/oxigraph-managed.js', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + startManagedOxigraph: mocks.startManagedOxigraph, + }; +}); + +vi.mock('../src/daemon/store-health-check.js', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + checkExternalStoreReachable: mocks.checkExternalStoreReachable, + checkOrSetStoreIdentity: mocks.checkOrSetStoreIdentity, + }; +}); + const { runDaemonInner } = await import('../src/daemon/lifecycle.js'); function closeDashboardDbFromAgentCreateArg(createArg: any): void { @@ -38,6 +58,29 @@ function closeDashboardDbFromAgentCreateArg(createArg: any): void { db?.close?.(); } +function managedOxigraphResult(dataDir: string) { + return { + handle: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + killSync: vi.fn(), + }, + storeConfig: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + managedByDkg: true, + }, + }, + largeLiteralStorage: { enabled: true, directory: join(dataDir, 'literal-blobs') }, + sharedMemoryPublicSnapshotStorage: { + enabled: true, + directory: join(dataDir, 'swm-public-snapshots'), + }, + }; +} + describe('daemon startup network validation', () => { let tempHome: string | undefined; let originalDkgHome: string | undefined; @@ -45,6 +88,15 @@ describe('daemon startup network validation', () => { let stderrWrite: typeof process.stderr.write = process.stderr.write; let uncaughtExceptionListeners: NodeJS.UncaughtExceptionListener[] = []; let unhandledRejectionListeners: NodeJS.UnhandledRejectionListener[] = []; + const originalAcceptStoreReset = process.env.DKG_ACCEPT_STORE_RESET; + + beforeEach(() => { + mocks.loadNetworkConfig.mockResolvedValue(undefined); + mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); + mocks.startManagedOxigraph.mockResolvedValue(null); + mocks.checkExternalStoreReachable.mockResolvedValue({ ok: true, backend: 'sparql-http', endpoint: 'http://127.0.0.1:12001/query' }); + mocks.checkOrSetStoreIdentity.mockResolvedValue({ ok: true, action: 'matched', nodeName: 'test-node' }); + }); afterEach(async () => { vi.restoreAllMocks(); @@ -64,10 +116,142 @@ describe('daemon startup network validation', () => { } else { process.env.DKG_HOME = originalDkgHome; } + if (originalAcceptStoreReset === undefined) { + delete process.env.DKG_ACCEPT_STORE_RESET; + } else { + process.env.DKG_ACCEPT_STORE_RESET = originalAcceptStoreReset; + } if (tempHome) await rm(tempHome, { recursive: true, force: true }); tempHome = undefined; }); + async function useTempHome(prefix: string) { + tempHome = await mkdtemp(join(tmpdir(), prefix)); + originalDkgHome = process.env.DKG_HOME; + process.env.DKG_HOME = tempHome; + stdoutWrite = process.stdout.write; + stderrWrite = process.stderr.write; + uncaughtExceptionListeners = process.listeners('uncaughtException') as NodeJS.UncaughtExceptionListener[]; + unhandledRejectionListeners = process.listeners('unhandledRejection') as NodeJS.UnhandledRejectionListener[]; + return vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + } + + it('exits before managed store startup when a blockless config has legacy store.nq and no reset acknowledgement', async () => { + const stdoutSpy = await useTempHome('dkg-legacy-store-gate-'); + await writeFile(join(tempHome!, 'store.nq'), '

.'); + vi + .spyOn(process, 'exit') + .mockImplementation(((code?: string | number | null) => { + throw new Error(`process.exit:${code}`); + }) as never); + + await expect(runDaemonInner(true, { + name: 'legacy-store-gate-test', + listenPort: 0, + nodeRole: 'edge', + } as any, Date.now())).rejects.toThrow('process.exit:1'); + + const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); + expect(output).toContain('legacy store.nq from the old implicit worker default'); + expect(output).toContain('DKG_ACCEPT_STORE_RESET=1'); + expect(mocks.startManagedOxigraph).not.toHaveBeenCalled(); + expect(mocks.agentCreate).not.toHaveBeenCalled(); + }); + + it('continues with the effective oxigraph-server store after legacy store.nq is acknowledged', async () => { + const stdoutSpy = await useTempHome('dkg-legacy-store-ack-'); + process.env.DKG_ACCEPT_STORE_RESET = '1'; + await writeFile(join(tempHome!, 'store.nq'), '

.'); + mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome!)); + mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); + + await expect(runDaemonInner(true, { + name: 'legacy-store-ack-test', + listenPort: 0, + nodeRole: 'edge', + } as any, Date.now())).rejects.toThrow('after-agent-create'); + + const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); + expect(output).toContain('using oxigraph-server'); + expect(mocks.startManagedOxigraph).toHaveBeenCalledTimes(1); + expect(mocks.startManagedOxigraph.mock.calls[0]?.[0]).toMatchObject({ + dataDir: tempHome, + config: { + store: { backend: 'oxigraph-server', options: {} }, + }, + }); + expect(mocks.agentCreate).toHaveBeenCalledTimes(1); + expect(mocks.agentCreate.mock.calls[0]?.[0]).toMatchObject({ + storeConfig: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + managedByDkg: true, + }, + }, + largeLiteralStorage: { enabled: true, directory: join(tempHome!, 'literal-blobs') }, + sharedMemoryPublicSnapshotStorage: { enabled: true, directory: join(tempHome!, 'swm-public-snapshots') }, + }); + }); + + it('blocks a wizard-rewritten oxigraph-server config with legacy store.nq and no backend marker', async () => { + const stdoutSpy = await useTempHome('dkg-rewritten-legacy-store-gate-'); + await writeFile(join(tempHome!, 'store.nq'), '

.'); + vi + .spyOn(process, 'exit') + .mockImplementation(((code?: string | number | null) => { + throw new Error(`process.exit:${code}`); + }) as never); + + await expect(runDaemonInner(true, { + name: 'rewritten-legacy-store-gate-test', + listenPort: 0, + nodeRole: 'edge', + store: { backend: 'oxigraph-server', options: {} }, + } as any, Date.now())).rejects.toThrow('process.exit:1'); + + const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); + expect(output).toContain('legacy store.nq from the old worker-backed store'); + expect(output).toContain('DKG_ACCEPT_STORE_RESET=1'); + expect(mocks.startManagedOxigraph).not.toHaveBeenCalled(); + expect(mocks.agentCreate).not.toHaveBeenCalled(); + }); + + it('migrates an explicit legacy worker config after reset acknowledgement', async () => { + await useTempHome('dkg-explicit-legacy-store-ack-'); + process.env.DKG_ACCEPT_STORE_RESET = '1'; + await writeFile(join(tempHome!, 'store.nq'), '

.'); + mocks.startManagedOxigraph.mockResolvedValue({ + handle: { queryEndpoint: 'http://127.0.0.1:12001/query', updateEndpoint: 'http://127.0.0.1:12001/update', killSync: vi.fn() }, + storeConfig: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + managedByDkg: true, + }, + }, + largeLiteralStorage: { enabled: true, directory: join(tempHome!, 'literal-blobs') }, + sharedMemoryPublicSnapshotStorage: { enabled: true, directory: join(tempHome!, 'swm-public-snapshots') }, + }); + mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); + + await expect(runDaemonInner(true, { + name: 'explicit-legacy-store-ack-test', + listenPort: 0, + nodeRole: 'edge', + store: { backend: 'oxigraph-worker' }, + } as any, Date.now())).rejects.toThrow('after-agent-create'); + + expect(mocks.startManagedOxigraph.mock.calls[0]?.[0]).toMatchObject({ + config: { store: { backend: 'oxigraph-server', options: {} } }, + }); + expect(mocks.agentCreate.mock.calls[0]?.[0]).toMatchObject({ + storeConfig: { backend: 'sparql-http' }, + }); + }); + it('exits before agent creation when the selected network is pre-deployment', async () => { tempHome = await mkdtemp(join(tmpdir(), 'dkg-predeployment-startup-')); originalDkgHome = process.env.DKG_HOME; @@ -160,6 +344,7 @@ describe('daemon startup network validation', () => { defaultNodeRole: 'edge', }); mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); + mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome)); mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); @@ -214,6 +399,7 @@ describe('daemon startup network validation', () => { defaultNodeRole: 'edge', }); mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); + mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome)); mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); diff --git a/packages/cli/test/daemon-state.test.ts b/packages/cli/test/daemon-state.test.ts new file mode 100644 index 0000000000..0c158fc8eb --- /dev/null +++ b/packages/cli/test/daemon-state.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + DAEMON_STATE_FILE, + readPersistedDaemonState, + readPersistedNetworkConfig, + readPersistedStoreBackend, + writePersistedChainResetMarker, + writePersistedNetworkConfig, + writePersistedStoreBackend, +} from '../src/daemon/daemon-state.js'; + +describe('persisted daemon state', () => { + it('preserves sibling fields when marker, backend, and network writers update independently', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-daemon-state-')); + + writePersistedStoreBackend(dataDir, 'oxigraph-server'); + writePersistedNetworkConfig(dataDir, 'mainnet-gnosis'); + writePersistedChainResetMarker(dataDir, 'reset-42'); + + expect(readPersistedDaemonState(dataDir)).toMatchObject({ + chainResetMarker: 'reset-42', + lastBackend: 'oxigraph-server', + lastNetworkConfig: 'mainnet-gnosis', + }); + expect(readPersistedStoreBackend(dataDir)).toBe('oxigraph-server'); + expect(readPersistedNetworkConfig(dataDir)).toBe('mainnet-gnosis'); + + writePersistedStoreBackend(dataDir, 'blazegraph'); + expect(readPersistedDaemonState(dataDir)).toMatchObject({ + chainResetMarker: 'reset-42', + lastBackend: 'blazegraph', + lastNetworkConfig: 'mainnet-gnosis', + }); + }); + + it('treats malformed state as absent and repairs it on the next write', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-daemon-state-invalid-')); + await writeFile( + join(dataDir, DAEMON_STATE_FILE), + JSON.stringify({ chainResetMarker: 42, lastBackend: 'oxigraph' }), + ); + + expect(readPersistedDaemonState(dataDir)).toBeNull(); + writePersistedNetworkConfig(dataDir, 'testnet'); + expect(readPersistedDaemonState(dataDir)).toMatchObject({ + chainResetMarker: null, + lastNetworkConfig: 'testnet', + }); + }); +}); diff --git a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts index 98840f7494..5934ab3650 100644 --- a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts +++ b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts @@ -148,6 +148,7 @@ describe('runDaemonInner StorageACK timing wiring', () => { networkConfig: 'mainnet-gnosis', listenPort: 0, nodeRole: 'core', + store: { backend: 'oxigraph' }, chain: { type: 'evm', rpcUrl: 'https://private-rpc.example', @@ -285,6 +286,7 @@ describe('runDaemonInner StorageACK timing wiring', () => { listenPort: 0, nodeRole: 'edge', apiPort: 0, + store: { backend: 'oxigraph' }, auth: { enabled: false }, promoteQueue: { enabled: false }, publisher: { enabled: true }, diff --git a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts index 07eb5d57b7..3f4eaa1636 100644 --- a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts +++ b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts @@ -114,6 +114,7 @@ describe('runDaemonInner wires sync options into DKGAgent.create', () => { networkConfig: 'mainnet-gnosis', listenPort: 0, nodeRole: 'core', + store: { backend: 'oxigraph' }, chain: { type: 'evm', rpcUrl: 'https://private-rpc.example', diff --git a/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts b/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts index c1617aa888..3204585822 100644 --- a/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts +++ b/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts @@ -62,7 +62,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: true }, store: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(home, 'store.nq') }, }, chain: { diff --git a/packages/cli/test/handle-request-store-persistence.test.ts b/packages/cli/test/handle-request-store-persistence.test.ts new file mode 100644 index 0000000000..0afd3d0305 --- /dev/null +++ b/packages/cli/test/handle-request-store-persistence.test.ts @@ -0,0 +1,103 @@ +import { createServer } from 'node:http'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { DkgConfig } from '../src/config.js'; +import { handleRequest } from '../src/daemon/handle-request.js'; +import { createRequestStoreContext } from '../src/daemon/routes/context.js'; +import type { StoreRuntimeContext } from '../src/daemon/store-runtime.js'; + +describe('handleRequest store config boundary', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('persists the operator config rather than the materialized managed-store runtime', async () => { + const home = await mkdtemp(join(tmpdir(), 'dkg-handle-request-store-')); + vi.stubEnv('DKG_HOME', home); + + const operatorConfig: DkgConfig = { + name: 'operator-config-route-test', + nodeRole: 'edge', + chain: { type: 'mock' }, + }; + const storeRuntime: StoreRuntimeContext = { + operatorConfig, + effectiveStore: { backend: 'oxigraph-server', options: {} }, + runtimeStore: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:7878/query', + updateEndpoint: 'http://127.0.0.1:7878/update', + managedByDkg: true, + }, + }, + }; + const requestStoreContext = createRequestStoreContext(storeRuntime); + expect(requestStoreContext).toEqual({ + config: operatorConfig, + effectiveStore: storeRuntime.effectiveStore, + runtimeStore: storeRuntime.runtimeStore, + }); + expect(requestStoreContext).not.toHaveProperty('operatorConfig'); + expect(requestStoreContext).not.toHaveProperty('storeRuntime'); + + const server = createServer((req, res) => { + const args: Parameters = [ + req, + res, + { resolveAgentAddress: () => 'operator-config-route-test' } as any, + {} as any, + null, + storeRuntime, + Date.now(), + {} as any, + { wallets: [] }, + null, + {} as any, + {} as any, + undefined, + '0.0.0-test', + '', + {} as any, + {} as any, + new Map(), + new Map(), + {} as any, + null, + new Set(), + '127.0.0.1', + { value: 0 }, + [], + { inFlight: 0, max: 0, rejectedTotal: 0 }, + ]; + void handleRequest(...args).catch(() => { + res.statusCode = 500; + res.end('route failed'); + }); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address() as AddressInfo; + const response = await fetch(`http://127.0.0.1:${address.port}/api/register-adapter`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: 'openclaw' }), + }); + expect(response.status).toBe(200); + + const persisted = JSON.parse(await readFile(join(home, 'config.json'), 'utf8')) as DkgConfig; + expect(persisted.store).toBeUndefined(); + expect(persisted.localAgentIntegrations?.openclaw?.enabled).toBe(true); + expect(JSON.stringify(persisted)).not.toContain('127.0.0.1:7878'); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + await rm(home, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/test/helpers/live-daemon.ts b/packages/cli/test/helpers/live-daemon.ts index 683b1ba5f9..217188bd0a 100644 --- a/packages/cli/test/helpers/live-daemon.ts +++ b/packages/cli/test/helpers/live-daemon.ts @@ -89,7 +89,7 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise ({})) as { code?: string; reason?: string }; - if (body.code !== 'async_publisher_unavailable') break; + const body = await res.json().catch(() => ({})) as { error?: string; reason?: string }; + if (body.error !== 'PublisherUnavailable' && body.error !== 'PublisherDisabled') break; if (body.reason !== 'publisher_starting') { throw new Error(`Async publisher failed readiness: ${body.reason ?? res.status}`); } diff --git a/packages/cli/test/oxigraph-managed.test.ts b/packages/cli/test/oxigraph-managed.test.ts index f069eb6676..cb89c9592f 100644 --- a/packages/cli/test/oxigraph-managed.test.ts +++ b/packages/cli/test/oxigraph-managed.test.ts @@ -86,7 +86,7 @@ afterAll(async () => { describe('planManagedOxigraph', () => { it('returns null for non-oxigraph-server backends', () => { - expect(planManagedOxigraph({ store: { backend: 'oxigraph-worker' } }, '/data')).toBeNull(); + expect(planManagedOxigraph({ store: { backend: 'oxigraph' } }, '/data')).toBeNull(); expect(planManagedOxigraph({ store: { backend: 'sparql-http' } }, '/data')).toBeNull(); expect(planManagedOxigraph({}, '/data')).toBeNull(); }); @@ -327,7 +327,7 @@ describe('startManagedOxigraph (real download + real server)', () => { // Contract: a non-managed backend is a no-op — null result, and nothing // observable happens (no cache dir is created, nothing binds a port). const result = await startManagedOxigraph({ - config: { store: { backend: 'oxigraph-worker' } }, + config: { store: { backend: 'oxigraph' } }, dataDir: '/data', }); expect(result).toBeNull(); diff --git a/packages/cli/test/publisher-managed-store.test.ts b/packages/cli/test/publisher-managed-store.test.ts new file mode 100644 index 0000000000..5e25109379 --- /dev/null +++ b/packages/cli/test/publisher-managed-store.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createPublisherInspector } from '../src/publisher-runner.js'; + +describe('daemon-down publisher inspection', () => { + it('requires the daemon for the managed oxigraph-server backend', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-managed-store-')); + + await expect(createPublisherInspector({ + dataDir, + config: { + name: 'managed-publisher-test', + nodeRole: 'edge', + store: { backend: 'oxigraph-server', options: {} }, + }, + })).rejects.toThrow( + /daemon-managed store "oxigraph-server" require a running DKG daemon.*Start the daemon and retry/, + ); + }); +}); diff --git a/packages/cli/test/publisher-wallets.test.ts b/packages/cli/test/publisher-wallets.test.ts index 10e2085a9b..3b73d221b3 100644 --- a/packages/cli/test/publisher-wallets.test.ts +++ b/packages/cli/test/publisher-wallets.test.ts @@ -144,6 +144,48 @@ describe('publisher wallets', () => { ).rejects.toThrow('dkg publisher wallet add '); }); + it('boots and closes the standalone publisher runtime with the persistent fallback when config has no store block', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-runtime-')); + const wallet = ethers.Wallet.createRandom(); + await addPublisherWallet(dataDir, wallet.privateKey); + + const runtime = await createPublisherRuntime({ + dataDir, + config: { + name: 'test-node', + apiPort: 9200, + listenPort: 0, + nodeRole: 'edge', + contextGraphs: [], + chain: { type: 'mock' }, + }, + pollIntervalMs: 10, + errorBackoffMs: 10, + }); + + await runtime.publisher.lift({ + swmId: 'swm-main', + shareOperationId: 'share-no-store-fallback', + roots: ['urn:local:/fallback'], + contextGraphId: 'music-social', + namespace: 'aloha', + scope: 'person-profile', + transitionType: 'CREATE', + authority: { type: 'owner', proofRef: 'proof:owner:fallback' }, + }); + + await runtime.stop(); + const persistentStore = await createTripleStore({ + backend: 'oxigraph-persistent', + options: { path: join(dataDir, 'store.nq') }, + }); + const inspector = createPublisherInspectorFromStore(persistentStore, true); + const jobs = await inspector.publisher.list(); + expect(jobs).toHaveLength(1); + expect(jobs[0]?.jobId).toBeDefined(); + await inspector.stop(); + }); + it('resolves publisher chain defaults from config.networkConfig', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-runtime-')); const wallet = ethers.Wallet.createRandom(); diff --git a/packages/cli/test/status-route-rpc.test.ts b/packages/cli/test/status-route-rpc.test.ts index ecc88c7d20..29507ed102 100644 --- a/packages/cli/test/status-route-rpc.test.ts +++ b/packages/cli/test/status-route-rpc.test.ts @@ -33,13 +33,35 @@ import { } from '@origintrail-official/dkg-chain'; import { computeNetworkId } from '../../core/src/genesis.js'; import { getSharedContext } from '../../chain/test/evm-test-context.js'; -import { loadNetworkConfig } from '../src/config.js'; -import { handleStatusRoutes } from '../src/daemon/routes/status.js'; -import type { RequestContext } from '../src/daemon/routes/context.js'; +import { loadNetworkConfig, type DkgConfig } from '../src/config.js'; +import { + handleStatusRoutes, + invalidateExternalStoreQuadsCache, +} from '../src/daemon/routes/status.js'; +import { + createRequestStoreContext, + type RequestContext, +} from '../src/daemon/routes/context.js'; import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from './helpers/live-daemon.js'; // A port nothing listens on — connecting to it is a REAL refused connection. const DEAD_RPC = 'http://127.0.0.1:9'; +const MANAGED_QUERY_ENDPOINT = 'http://127.0.0.1:7878/query'; + +function managedStoreRuntime(operatorConfig: DkgConfig) { + return { + operatorConfig, + effectiveStore: { backend: 'oxigraph-server', options: {} }, + runtimeStore: { + backend: 'sparql-http', + options: { + queryEndpoint: MANAGED_QUERY_ENDPOINT, + updateEndpoint: 'http://127.0.0.1:7878/update', + managedByDkg: true, + }, + }, + }; +} describe('/api/status + /api/chain/rpc-health (real daemon, real chain)', () => { let daemon: LiveDaemon; @@ -118,9 +140,124 @@ describe('/api/status + /api/chain/rpc-health (real daemon, real chain)', () => }); describe('/api/status selected overlay details', () => { + it('reports the effective managed store for a blockless config', async () => { + const config: DkgConfig = { + name: 'status-blockless-store-test', + nodeRole: 'edge', + chain: { type: 'mock' }, + }; + const query = vi.fn(async () => ({ + type: 'bindings' as const, + bindings: [{ c: '42' }], + })); + invalidateExternalStoreQuadsCache(); + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + await handleStatusRoutes({ + req, + res, + path: url.pathname, + url, + network: null, + ...createRequestStoreContext(managedStoreRuntime(config)), + startedAt: Date.now(), + agent: { + peerId: 'peer-status-test', + multiaddrs: [], + node: { + libp2p: { getConnections: () => [] }, + getRelayStats: () => null, + }, + publisher: { getIdentityId: () => 0n }, + store: { query }, + }, + nodeVersion: '0.0.0-test', + nodeCommit: '', + admission: { inFlight: 0, max: 0, rejectedTotal: 0 }, + } as unknown as RequestContext); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address() as AddressInfo; + const res = await fetch(`http://127.0.0.1:${address.port}/api/status`); + expect(res.status).toBe(200); + const body: any = await res.json(); + + expect(body.storeBackend).toBe('oxigraph-server'); + expect(body.storeUrl).toBe(MANAGED_QUERY_ENDPOINT); + expect(body.storeQuads).toBe(42); + expect(query).toHaveBeenCalledOnce(); + } finally { + invalidateExternalStoreQuadsCache(); + await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); + } + }); + + it('reports the effective managed store after an acknowledged worker cutover', async () => { + const config: DkgConfig = { + name: 'status-worker-cutover-test', + nodeRole: 'edge', + chain: { type: 'mock' }, + store: { backend: 'oxigraph-worker', options: {} }, + }; + const query = vi.fn(async () => ({ + type: 'bindings' as const, + bindings: [{ c: '17' }], + })); + invalidateExternalStoreQuadsCache(); + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + await handleStatusRoutes({ + req, + res, + path: url.pathname, + url, + network: null, + ...createRequestStoreContext(managedStoreRuntime(config)), + startedAt: Date.now(), + agent: { + peerId: 'peer-status-test', + multiaddrs: [], + node: { + libp2p: { getConnections: () => [] }, + getRelayStats: () => null, + }, + publisher: { getIdentityId: () => 0n }, + store: { query }, + }, + nodeVersion: '0.0.0-test', + nodeCommit: '', + admission: { inFlight: 0, max: 0, rejectedTotal: 0 }, + } as unknown as RequestContext); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address() as AddressInfo; + const res = await fetch(`http://127.0.0.1:${address.port}/api/status`); + expect(res.status).toBe(200); + const body: any = await res.json(); + + expect(body.storeBackend).toBe('oxigraph-server'); + expect(body.storeUrl).toBe(MANAGED_QUERY_ENDPOINT); + expect(body.storeQuads).toBe(17); + expect(query).toHaveBeenCalledOnce(); + } finally { + invalidateExternalStoreQuadsCache(); + await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); + } + }); + it('returns the network id and name for the selected overlay genesis', async () => { const network = await loadNetworkConfig('mainnet-gnosis'); expect(network).not.toBeNull(); + const config: DkgConfig = { + name: 'status-selected-overlay-test', + networkConfig: 'mainnet-gnosis', + nodeRole: 'edge', + chain: { type: 'mock' }, + }; const server = createServer(async (req, res) => { const url = new URL(req.url ?? '/', 'http://127.0.0.1'); @@ -130,12 +267,7 @@ describe('/api/status selected overlay details', () => { path: url.pathname, url, network, - config: { - name: 'status-selected-overlay-test', - networkConfig: 'mainnet-gnosis', - nodeRole: 'edge', - chain: { type: 'mock' }, - }, + ...createRequestStoreContext(managedStoreRuntime(config)), startedAt: Date.now(), agent: { peerId: 'peer-status-test', @@ -175,6 +307,17 @@ describe('/api/status selected overlay details', () => { const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const network = await loadNetworkConfig('mainnet-gnosis'); + const config: DkgConfig = { + name: 'status-failover-counter-test', + networkConfig: 'mainnet-gnosis', + nodeRole: 'edge', + chain: { + type: 'evm', + rpcUrl: 'http://127.0.0.1:9', + hubAddress: `0x${'ab'.repeat(20)}`, + chainId: 'evm:31337', + }, + }; try { // Seed the process-wide failover counters the status route reads, then // assert /api/status reflects the exact delta. This would FAIL if the @@ -195,17 +338,7 @@ describe('/api/status selected overlay details', () => { path: url.pathname, url, network, - config: { - name: 'status-failover-counter-test', - networkConfig: 'mainnet-gnosis', - nodeRole: 'edge', - chain: { - type: 'evm', - rpcUrl: 'http://127.0.0.1:9', - hubAddress: `0x${'ab'.repeat(20)}`, - chainId: 'evm:31337', - }, - }, + ...createRequestStoreContext(managedStoreRuntime(config)), startedAt: Date.now(), agent: { peerId: 'peer-status-test', diff --git a/packages/cli/test/store-backend-taxonomy.test.ts b/packages/cli/test/store-backend-taxonomy.test.ts new file mode 100644 index 0000000000..c904cb28c2 --- /dev/null +++ b/packages/cli/test/store-backend-taxonomy.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + STORAGE_ADAPTERS, + classifyTripleStoreBackend, + customTripleStoreBackend, + isExternalBackend, + isStorageAdapterBackend, + storageAdapterNames, +} from '@origintrail-official/dkg-storage'; +import { validateStoreConfig, type DkgConfig } from '../src/config.js'; +import { + DEFAULT_DAEMON_STORE_BACKEND, + MANAGED_DAEMON_STORE_BACKEND, + STORE_BACKENDS, + configBackendNames, + isManagedLocalBackend, + isRetiredStoreBackend, + requireStorageAdapterBackend, + storeFlagBackendNames, + storeBackendNames, + wizardBackendChoices, + type StoreBackend, +} from '../src/store-backends.js'; +import { checkExternalStoreReachable } from '../src/daemon/store-health-check.js'; +import { planManagedOxigraph } from '../src/daemon/oxigraph-managed.js'; +import { storeBackendHasStatusHealth } from '../src/daemon/routes/status.js'; + +function configForBackend(backend: StoreBackend): DkgConfig { + const policy = STORE_BACKENDS[backend]; + const options = policy.kind === 'external' + ? { [policy.queryEndpointOption]: 'http://store.test/query' } + : {}; + return { + name: 'taxonomy-test', + apiPort: 9200, + listenPort: 4001, + nodeRole: 'edge', + store: { backend, options }, + } as DkgConfig; +} + +describe('canonical store backend taxonomy', () => { + it('drives config validation and wizard discovery for every registered backend', () => { + const configBackends = configBackendNames(); + const wizardBackends = wizardBackendChoices(); + const flagBackends = storeFlagBackendNames(); + + for (const backend of storeBackendNames()) { + const policy = STORE_BACKENDS[backend]; + const errors = validateStoreConfig(configForBackend(backend)); + + expect((configBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired); + expect((wizardBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired && policy.wizard); + expect((flagBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired && policy.storeFlag); + expect(errors.some((error) => error.field === 'store.backend'), backend).toBe(policy.retired); + if (!policy.retired) expect(errors, backend).toEqual([]); + if (policy.wizard) expect('label' in policy && policy.label.length > 0, backend).toBe(true); + } + }); + + it('keeps daemon health, managed startup, and status routing aligned for every backend', async () => { + for (const backend of storeBackendNames()) { + const policy = STORE_BACKENDS[backend]; + const external = policy.kind === 'external'; + const managed = policy.kind === 'managed-local'; + + expect(isExternalBackend(backend), backend).toBe(external); + expect(isManagedLocalBackend(backend), backend).toBe(managed); + expect(isRetiredStoreBackend(backend), backend).toBe(policy.retired); + expect(isStorageAdapterBackend(backend), backend).toBe(policy.adapter); + expect(classifyTripleStoreBackend(backend).kind, backend).toBe( + policy.adapter ? 'adapter' : 'custom', + ); + expect(storeBackendHasStatusHealth(backend), backend).toBe(external || managed); + expect(planManagedOxigraph(configForBackend(backend), '/data') !== null, backend).toBe(managed); + + const fetch = vi.fn(async () => new Response('{}', { status: 200 })); + const health = await checkExternalStoreReachable({ + storeConfig: configForBackend(backend).store, + fetch, + }); + expect(health.ok, backend).toBe(true); + expect(fetch.mock.calls.length > 0, backend).toBe(external); + } + }); + + it('derives the daemon default and managed-local constant from the registry', () => { + expect(DEFAULT_DAEMON_STORE_BACKEND).toBe(MANAGED_DAEMON_STORE_BACKEND); + expect(STORE_BACKENDS[DEFAULT_DAEMON_STORE_BACKEND]).toMatchObject({ + default: true, + kind: 'managed-local', + retired: false, + }); + }); + + it('composes daemon policy over the storage-owned adapter registry', () => { + expect(storageAdapterNames()).toEqual([ + 'oxigraph', + 'oxigraph-persistent', + 'blazegraph', + 'sparql-http', + ]); + expect(storageAdapterNames()).not.toContain('oxigraph-server'); + expect(storageAdapterNames()).not.toContain('oxigraph-worker'); + for (const backend of storageAdapterNames()) { + expect(STORE_BACKENDS[backend]).toMatchObject(STORAGE_ADAPTERS[backend]); + expect(requireStorageAdapterBackend(backend)).toBe(backend); + } + expect(() => requireStorageAdapterBackend('oxigraph-server')).toThrow( + /not a constructible storage adapter/, + ); + }); + + it('requires an explicit custom-backend escape hatch outside the known registry', () => { + const backend = customTripleStoreBackend('vendor-plugin-store'); + expect(classifyTripleStoreBackend(backend)).toEqual({ + kind: 'custom', + backend: 'vendor-plugin-store', + }); + expect(() => customTripleStoreBackend('oxigraph')).toThrow(/known triple-store adapter/i); + }); +}); diff --git a/packages/cli/test/store-health-check.test.ts b/packages/cli/test/store-health-check.test.ts index b01d80ba5a..49797a2ea8 100644 --- a/packages/cli/test/store-health-check.test.ts +++ b/packages/cli/test/store-health-check.test.ts @@ -37,7 +37,7 @@ describe('checkExternalStoreReachable', () => { it('passes through with no I/O for local backends', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await checkExternalStoreReachable({ - storeConfig: { backend: 'oxigraph-worker' }, + storeConfig: { backend: 'oxigraph' }, fetch: fn, }); expect(result.ok).toBe(true); diff --git a/packages/cli/test/store-identity-tag.test.ts b/packages/cli/test/store-identity-tag.test.ts index fd8b19d354..f438be62d2 100644 --- a/packages/cli/test/store-identity-tag.test.ts +++ b/packages/cli/test/store-identity-tag.test.ts @@ -43,7 +43,7 @@ describe('checkOrSetStoreIdentity', () => { it('skips for local oxigraph backend', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await checkOrSetStoreIdentity({ - storeConfig: { backend: 'oxigraph-worker' }, + storeConfig: { backend: 'oxigraph' }, nodeName: 'mynode', fetch: fn, }); diff --git a/packages/cli/test/store-runtime.test.ts b/packages/cli/test/store-runtime.test.ts new file mode 100644 index 0000000000..15fbdab1b9 --- /dev/null +++ b/packages/cli/test/store-runtime.test.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + resolveDaemonStoreBootPlan, + resolveDaemonStoreRuntime, + type DaemonStoreBootDecision, + type DaemonStoreBootPlan, +} from '../src/daemon/store-runtime.js'; +import { saveConfig, type DkgConfig } from '../src/config.js'; +import type { StorageAdapterBackend } from '@origintrail-official/dkg-storage'; + +function mk(overrides: Partial = {}): DkgConfig { + return { + name: 'dkg-node', + apiPort: 9200, + listenPort: 4001, + nodeRole: 'edge', + ...overrides, + } as DkgConfig; +} + +function expectBootable( + decision: DaemonStoreBootDecision, +): asserts decision is DaemonStoreBootPlan { + expect(decision.kind).toBe('bootable'); + if (decision.kind !== 'bootable') { + throw new Error(`Expected a bootable store plan, got ${decision.kind}`); + } +} + +describe('resolveDaemonStoreBootPlan', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('keeps blockless operator config separate from the materialized daemon store default', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-')); + const config = mk(); + + const plan = resolveDaemonStoreBootPlan({ + config, + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + expect(plan.operatorConfig).toBe(config); + expect(plan.operatorConfig.store).toBeUndefined(); + expect(plan.effectiveConfig.store).toEqual({ backend: 'oxigraph-server', options: {} }); + expect(plan.effectiveStore).toEqual({ backend: 'oxigraph-server', options: {} }); + }); + + it('fails at runtime-plan construction when a managed store was not materialized', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-unmaterialized-')); + const plan = resolveDaemonStoreBootPlan({ + config: mk(), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + expect(() => resolveDaemonStoreRuntime(plan, null)).toThrow( + /oxigraph-server.*not materialized to a storage adapter/, + ); + }); + + it('refines a live runtime store to a constructible adapter backend', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-adapter-')); + const plan = resolveDaemonStoreBootPlan({ + config: mk({ store: { backend: 'oxigraph', options: {} } }), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + const runtime = resolveDaemonStoreRuntime(plan, null); + expect(runtime.runtimeStore.backend).toBe('oxigraph'); + expectTypeOf(runtime.runtimeStore.backend).toMatchTypeOf(); + }); + + it('does not persist the materialized store default during an unrelated config save', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-save-')); + vi.stubEnv('DKG_HOME', dataDir); + const plan = resolveDaemonStoreBootPlan({ + config: mk(), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + plan.operatorConfig.sharedMemoryTtlMs = 1234; + await saveConfig(plan.operatorConfig); + + const persisted = JSON.parse(await readFile(join(dataDir, 'config.json'), 'utf8')) as DkgConfig; + expect(persisted.sharedMemoryTtlMs).toBe(1234); + expect(persisted.store).toBeUndefined(); + expect(plan.effectiveConfig.store).toEqual({ backend: 'oxigraph-server', options: {} }); + }); + + it('gates an oxigraph-server cutover when legacy store.nq exists without a backend marker', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-cutover-')); + await writeFile(join(dataDir, 'store.nq'), '

.'); + + const plan = resolveDaemonStoreBootPlan({ + config: mk({ store: { backend: 'oxigraph-server', options: {} } }), + dataDir, + acceptStoreReset: false, + }); + + expect(plan.kind).toBe('blocked-legacy-cutover'); + if (plan.kind !== 'blocked-legacy-cutover') { + throw new Error(`Expected a blocked legacy cutover, got ${plan.kind}`); + } + expect(plan.message).toContain('DKG_ACCEPT_STORE_RESET=1'); + }); + + it('does not repeatedly gate a cutover already recorded as oxigraph-server', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-recorded-')); + await writeFile(join(dataDir, 'store.nq'), '

.'); + await writeFile( + join(dataDir, '.network-state.json'), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-server', savedAt: Date.now() }), + ); + + const plan = resolveDaemonStoreBootPlan({ + config: mk(), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + expect(plan.effectiveStore.backend).toBe('oxigraph-server'); + }); + + it('migrates an explicit worker config only after acknowledging its legacy store', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-worker-')); + await writeFile(join(dataDir, 'store.nq'), '

.'); + const config = mk({ store: { backend: 'oxigraph-worker' } }); + + const blocked = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: false }); + expect(blocked.kind).toBe('blocked-legacy-cutover'); + if (blocked.kind !== 'blocked-legacy-cutover') { + throw new Error(`Expected a blocked legacy cutover, got ${blocked.kind}`); + } + expect(blocked.message).toContain('legacy store.nq from the old oxigraph-worker backend'); + + const acknowledged = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: true }); + expectBootable(acknowledged); + expect(acknowledged.effectiveStore).toEqual({ backend: 'oxigraph-server', options: {} }); + expect(acknowledged.operatorConfig).toBe(config); + expect(acknowledged.operatorConfig.store?.backend).toBe('oxigraph-worker'); + }); + + it('classifies an explicit worker config without legacy data as invalid', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-invalid-worker-')); + const config = mk({ store: { backend: 'oxigraph-worker' } }); + + const decision = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: false }); + + expect(decision).toEqual({ kind: 'invalid-config', operatorConfig: config }); + }); +}); diff --git a/packages/cli/test/store-wizard.test.ts b/packages/cli/test/store-wizard.test.ts index 02a2ac794a..a67557bba3 100644 --- a/packages/cli/test/store-wizard.test.ts +++ b/packages/cli/test/store-wizard.test.ts @@ -10,7 +10,7 @@ * - Blazegraph + valid URL: store block persisted with * `managedByDkg: false`. * - Blazegraph + unreachable URL: surfaces formatted failure, allows - * retry, abort returns to local default. + * retry, abort leaves the managed local default in place. * - Blazegraph + 404 URL: namespace-missing branch fires; message * mentions namespace, not network. * - Blank URL prompt: PR 2's "no Docker yet" message + retry. @@ -28,6 +28,14 @@ import { describe, it, expect } from 'vitest'; import { applyStoreFlagsToConfig, promptStoreBackend } from '../src/store-wizard.js'; import type { DkgConfig } from '../src/config.js'; +import { + STORE_BACKENDS, + configBackendList, + configBackendNames, + storeFlagBackendList, + storeFlagBackendNames, + wizardBackendChoices, +} from '../src/store-backends.js'; function mockFetch(handler: (input: any, init?: any) => Response | Promise) { const calls: Array<{ url: string; init?: RequestInit }> = []; @@ -54,6 +62,26 @@ function mockAsk(scriptedAnswers: string[]): (q: string, def?: string) => Promis // --------------------------------------------------------------------- describe('promptStoreBackend', () => { + it('keeps config, wizard, and flag backend lists aligned with their policies', () => { + expect(configBackendList().split(', ')).toEqual(configBackendNames()); + expect(configBackendNames()).toEqual([ + 'oxigraph-server', + 'oxigraph', + 'oxigraph-persistent', + 'blazegraph', + 'sparql-http', + ]); + expect(storeFlagBackendList().split(', ')).toEqual(storeFlagBackendNames()); + expect(storeFlagBackendNames()).toEqual([ + 'oxigraph-server', + 'oxigraph', + 'blazegraph', + 'sparql-http', + ]); + expect(wizardBackendChoices()).toEqual(['oxigraph-server', 'oxigraph', 'blazegraph']); + expect(Object.keys(STORE_BACKENDS)).toContain('oxigraph-worker'); + }); + it('defaults to oxigraph-server when the operator accepts the default', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await promptStoreBackend({ @@ -65,31 +93,26 @@ describe('promptStoreBackend', () => { expect(calls).toHaveLength(0); // no URL probe issued for a local backend }); - it('returns no store block (embedded worker) when operator picks "oxigraph" by name', async () => { + it('persists an explicit embedded development store when operator picks "oxigraph" by name', async () => { const result = await promptStoreBackend({ ask: mockAsk(['oxigraph']), log: () => {}, }); - expect(result.storeBlock).toBeNull(); + expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); }); - it('returns no store block (embedded worker) when operator picks the worker by number', async () => { + it('persists an explicit embedded development store when operator picks oxigraph by number', async () => { // Menu is now `1) oxigraph-server 2) oxigraph 3) blazegraph` — picking - // option 2 must opt down to the embedded in-process worker (no block). + // option 2 opts into the embedded in-memory store explicitly. const result = await promptStoreBackend({ ask: mockAsk(['2']), log: () => {}, }); - expect(result.storeBlock).toBeNull(); + expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); }); - it('preserves an explicit embedded backend verbatim on Enter-through (no flip, no option loss)', async () => { - // Codex #946 — only a *block-less* config should fall through to the new - // oxigraph-server default. A node that explicitly chose a local worker - // variant must keep it on a re-init Enter-through, AND keep its custom - // `options` (e.g. the worker's `options.path`): returning `null` would let - // `dkg init` write `store: undefined` and relocate the store on next boot. - for (const backend of ['oxigraph', 'oxigraph-worker', 'oxigraph-persistent'] as const) { + it('preserves a supported explicit embedded backend verbatim on Enter-through (no flip, no option loss)', async () => { + for (const backend of ['oxigraph', 'oxigraph-persistent'] as const) { const existingStore = { backend, options: { path: '/custom/store' } }; const result = await promptStoreBackend({ ask: mockAsk(['']), // Enter @@ -100,11 +123,7 @@ describe('promptStoreBackend', () => { } }); - it('switches to the default embedded worker when an oxigraph-persistent node EXPLICITLY picks oxigraph', async () => { - // Codex #946 — preservation must be gated on a true keep. An operator who - // explicitly selects option `2` / "oxigraph" to move a worker/persistent - // node back to the plain embedded default must NOT have the old backend + - // options silently retained. Both the numeric and named selection switch. + it('does not preserve oxigraph-persistent options when the operator explicitly picks oxigraph', async () => { const existingStore = { backend: 'oxigraph-persistent', options: { path: '/custom/store' } }; for (const answer of ['2', 'oxigraph']) { const result = await promptStoreBackend({ @@ -112,13 +131,33 @@ describe('promptStoreBackend', () => { existingStore, log: () => {}, }); - expect(result.storeBlock).toBeNull(); + expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); } }); + it('does not preserve retired oxigraph-worker configs on Enter-through', async () => { + const logs: string[] = []; + const result = await promptStoreBackend({ + ask: mockAsk(['']), + existingStore: { backend: 'oxigraph-worker', options: { path: '/custom/store' } }, + log: (m) => logs.push(m), + }); + expect(result.storeBlock).toEqual({ backend: 'oxigraph-server', options: {} }); + expect(logs.join('\n')).toMatch(/retired/); + }); + + it('rejects oxigraph-worker when typed explicitly', async () => { + await expect( + promptStoreBackend({ + ask: mockAsk(['oxigraph-worker']), + log: () => {}, + }), + ).rejects.toThrow(/no longer supported/); + }); + it('falls back to the recommended default (oxigraph-server) on an out-of-range number', async () => { // Codex #946 — a typo'd digit ("9") must not silently downgrade a fresh - // install to the embedded worker; it resolves to defaultBackend (option 1). + // install to an embedded backend; it resolves to defaultBackend (option 1). const result = await promptStoreBackend({ ask: mockAsk(['9']), log: () => {}, @@ -172,7 +211,7 @@ describe('promptStoreBackend', () => { expect(logs.some((l) => l.includes('STORE-HEALTH'))).toBe(true); }); - it('aborts to local default when operator declines retry on unreachable URL', async () => { + it('aborts to the managed local default when operator declines retry on unreachable URL', async () => { const { fn } = mockFetch(() => new Response('boom', { status: 500 })); const logs: string[] = []; const result = await promptStoreBackend({ @@ -283,7 +322,7 @@ describe('promptStoreBackend', () => { 'blazegraph', '', // blank URL 'n', // decline Docker - 'n', // decline retry-with-URL → abort to local default + 'n', // decline retry-with-URL → abort to managed local default ]), isDockerAvailable: async () => true, provisionBlazegraphDocker: async () => { @@ -616,7 +655,22 @@ describe('applyStoreFlagsToConfig', () => { storeFlag: 'neptune', log: () => {}, }), - ).rejects.toThrow(/oxigraph, blazegraph, sparql-http/); + ).rejects.toThrow(/oxigraph-server, oxigraph, blazegraph, sparql-http/); + }); + + it('does not advertise or accept oxigraph-persistent as a pathless --store choice', async () => { + const store = newMockConfig({ + ...baseConfig, + store: { backend: 'oxigraph-persistent', options: { path: '/existing/store.nq' } }, + } as DkgConfig); + const io = mockConfigIO(store); + + await expect(applyStoreFlagsToConfig({ + ...io, + storeFlag: 'oxigraph-persistent', + log: () => {}, + })).rejects.toThrow(/--store must be one of: oxigraph-server, oxigraph, blazegraph, sparql-http/); + expect(store.saved).toEqual([]); }); it('persists a daemon-managed oxigraph-server block (no URL required)', async () => { @@ -654,7 +708,7 @@ describe('applyStoreFlagsToConfig', () => { expect(store.saved[0].store).toEqual({ backend: 'oxigraph-server', options: {} }); }); - it('clears existing store block when --store oxigraph is passed', async () => { + it('persists an explicit oxigraph block when --store oxigraph is passed', async () => { const store = newMockConfig({ ...baseConfig, store: { @@ -669,10 +723,10 @@ describe('applyStoreFlagsToConfig', () => { log: () => {}, }); expect(store.saved).toHaveLength(1); - expect(store.saved[0].store).toBeUndefined(); + expect(store.saved[0].store).toEqual({ backend: 'oxigraph', options: {} }); }); - it('is a no-op when --store oxigraph is passed and no existing store block', async () => { + it('persists oxigraph when --store oxigraph is passed and no existing store block', async () => { const store = newMockConfig(baseConfig); const io = mockConfigIO(store); await applyStoreFlagsToConfig({ @@ -680,6 +734,20 @@ describe('applyStoreFlagsToConfig', () => { storeFlag: 'oxigraph', log: () => {}, }); + expect(store.saved).toHaveLength(1); + expect(store.saved[0].store).toEqual({ backend: 'oxigraph', options: {} }); + }); + + it('rejects --store oxigraph-worker', async () => { + const store = newMockConfig(baseConfig); + const io = mockConfigIO(store); + await expect( + applyStoreFlagsToConfig({ + ...io, + storeFlag: 'oxigraph-worker', + log: () => {}, + }), + ).rejects.toThrow(/no longer supported/); expect(store.saved).toEqual([]); }); diff --git a/packages/cli/test/validate-store-config.test.ts b/packages/cli/test/validate-store-config.test.ts index 39ff79e24c..fefd180a61 100644 --- a/packages/cli/test/validate-store-config.test.ts +++ b/packages/cli/test/validate-store-config.test.ts @@ -8,8 +8,8 @@ * when paired with an external backend (no local store path to * infer from). * - * Local backends (default Oxigraph) are unaffected — the function is a - * no-op for them. + * Supported local backends are unaffected. The retired `oxigraph-worker` + * backend is rejected before boot. * * Plan: `.cursor/plans/blazegraph_v10_support_178da670.plan.md` §PR 1 item 6. */ @@ -33,9 +33,12 @@ describe('validateStoreConfig', () => { }); describe('local backends', () => { - it('no-op for oxigraph-worker', () => { + it('no-op for supported local backends', () => { expect( - validateStoreConfig(mk({ store: { backend: 'oxigraph-worker' } })), + validateStoreConfig(mk({ store: { backend: 'oxigraph' } })), + ).toEqual([]); + expect( + validateStoreConfig(mk({ store: { backend: 'oxigraph-persistent', options: { path: '/tmp/store.nq' } } })), ).toEqual([]); }); @@ -44,10 +47,19 @@ describe('validateStoreConfig', () => { // if the backend is local; the wipe + health check honour // isExternalBackend the same way. const errors = validateStoreConfig( - mk({ store: { backend: 'oxigraph-worker', options: { url: 'irrelevant' } } }), + mk({ store: { backend: 'oxigraph', options: { url: 'irrelevant' } } }), ); expect(errors).toEqual([]); }); + + it('rejects the retired oxigraph-worker backend', () => { + const errors = validateStoreConfig( + mk({ store: { backend: 'oxigraph-worker' } }), + ); + expect(errors).toHaveLength(1); + expect(errors[0].field).toBe('store.backend'); + expect(errors[0].message).toMatch(/no longer supported/); + }); }); describe('blazegraph', () => { @@ -136,7 +148,7 @@ describe('validateStoreConfig', () => { it('does not enforce the directory requirement for local backends', () => { const errors = validateStoreConfig( mk({ - store: { backend: 'oxigraph-worker' }, + store: { backend: 'oxigraph-persistent', options: { path: '/tmp/store.nq' } }, largeLiteralStorage: { enabled: true }, }), ); diff --git a/packages/cli/test/write-preflight-resilience.test.ts b/packages/cli/test/write-preflight-resilience.test.ts index 3428effdca..2e3c44dca4 100644 --- a/packages/cli/test/write-preflight-resilience.test.ts +++ b/packages/cli/test/write-preflight-resilience.test.ts @@ -12,9 +12,9 @@ // `contextGraphActivePublicOnChainFromRegistry` / `contextGraphExists` // methods (invoked via ContextGraphResolveMethods.prototype on a narrow // harness carrying real state, the same shape the DKGAgent mixin sees), and -// • a REAL OxigraphWorkerStore that has been close()d — the honest -// "the store is closed" failure a crashed/closed worker produces live — -// alongside a healthy in-memory worker store for the no-change paths, and +// • a REAL SparqlHttpStore pointed at an unavailable loopback endpoint — the +// honest connection failure an unavailable external/managed store produces +// live — alongside a healthy embedded store for the no-change paths, and // • the REAL `resolveRequiredWriteContextGraphId` resolver with a real // captured ServerResponse sink (same conventions as // context-graph-write-path-validation.test.ts). @@ -38,7 +38,7 @@ import { WRITE_PREFLIGHT_CHAIN_RESCUE_TIMEOUT_MS, } from '../src/daemon/http-utils.js'; import { ContextGraphResolveMethods } from '../../agent/src/dkg-agent-cg-resolve.js'; -import { OxigraphWorkerStore, createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; +import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { DKG_ONTOLOGY, SYSTEM_CONTEXT_GRAPHS, @@ -104,8 +104,8 @@ function agentHarness( // The narrow data interface resolveRequiredWriteContextGraphId consumes. // `listContextGraphs` performs a real read against the harness store so a -// closed store rejects with the genuine worker error (the same failure mode -// the real list path hits), and a healthy store returns the given rows. +// unavailable store rejects with a genuine adapter error (the same failure +// mode the real list path hits), and a healthy store returns the given rows. function providerFor(harness: any, rows: Array> = []) { const listCalls: Array = []; return { @@ -153,25 +153,30 @@ function captureRes(): { res: ServerResponse; out: { status?: number; body?: any // The exact legacy 503 body the pre-resilience code produced when both legs // threw. Live callers and tests grep for this — the rescue must leave it // byte-compatible whenever it does NOT accept. -const LEGACY_503_ERROR = /^Failed to validate contextGraphId against known context graphs: exact preflight failed: .*store is closed.*; list validation failed: .*store is closed/; +const LEGACY_503_ERROR = /^Failed to validate contextGraphId against known context graphs: exact preflight failed: .*fetch failed.*; list validation failed: .*fetch failed/; -let closedStore: OxigraphWorkerStore; -let healthyStore: OxigraphWorkerStore; +let closedStore: TripleStore; +let healthyStore: TripleStore; beforeAll(async () => { - // A real worker-backed store that has been closed: every subsequent call - // rejects with the real `oxigraph-worker: cannot run "…" — the store is - // closed.` error — the exact live failure this track fixes. - closedStore = new OxigraphWorkerStore(undefined); - await closedStore.close(); - healthyStore = new OxigraphWorkerStore(undefined); + // A real external-store adapter pointed at an unavailable endpoint: every + // query rejects through the same fetch path used by managed/external stores. + closedStore = await createTripleStore({ + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:65535/query', + updateEndpoint: 'http://127.0.0.1:65535/update', + }, + }); + healthyStore = await createTripleStore({ backend: 'oxigraph' }); }); afterAll(async () => { + await closedStore.close(); await healthyStore.close(); }); -describe('probeContextGraphWritePreflight — store-failure resilience (real probe, real closed store)', () => { +describe('probeContextGraphWritePreflight — store-failure resilience (real probe, unavailable store)', () => { it('detects local content from indexed graph names while ignoring bookkeeping-only graphs', async () => { const store = await createTripleStore({ backend: 'oxigraph' }); const listGraphsByPrefix = store.listGraphsByPrefix?.bind(store); @@ -230,7 +235,7 @@ describe('probeContextGraphWritePreflight — store-failure resilience (real pro ); const probe = await harness.probeContextGraphWritePreflight(CG); expect(probe.storeUnavailable).toBe(true); - expect(probe.storeErrorMessage).toMatch(/store is closed/); + expect(probe.storeErrorMessage).toMatch(/fetch failed/); // In-memory registry state needs zero store I/O and must survive. expect(probe.inMemorySubscription).toEqual({ subscribed: true, synced: true }); // Store-derived facts are UNKNOWN — a store outage must never be @@ -269,7 +274,7 @@ describe('probeContextGraphWritePreflight — store-failure resilience (real pro }); }); -describe('resolveRequiredWriteContextGraphId — both-legs-failed rescue (real resolver, real closed store)', () => { +describe('resolveRequiredWriteContextGraphId — both-legs-failed rescue (real resolver, unavailable store)', () => { it('(a) ACCEPTS on positive on-chain proof the CG is active AND PUBLIC', async () => { const isActive = recorder(async (id: bigint) => id === 7n); const getAccessPolicy = recorder(async (_id: bigint) => 0); // 0 = public diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index 9fbdf64fbf..3372b91cd1 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -87,8 +87,8 @@ export default defineConfig({ // #761 — context graph write-target validation (from main). 'test/context-graph-write-path-validation.test.ts', // Track B — write-preflight resilience when the local store is - // slow/closed. Real resolver + real agent probe + real (closed) - // OxigraphWorkerStore; no hardhat needed. + // slow/unavailable. Real resolver + real agent probe + real + // unavailable SPARQL adapter; no hardhat needed. 'test/write-preflight-resilience.test.ts', 'test/http-literal-size-validation.test.ts', // CLI subprocess smoke with stub daemon only; no hardhat needed. diff --git a/packages/core/src/proto/storage-ack.ts b/packages/core/src/proto/storage-ack.ts index 11b2fafc61..d22b8d9e92 100644 --- a/packages/core/src/proto/storage-ack.ts +++ b/packages/core/src/proto/storage-ack.ts @@ -111,8 +111,8 @@ export const STORAGE_ACK_DECLINE_CODES = { /** * The core hit a peer-LOCAL, transient infrastructure failure while * servicing an otherwise well-formed request: its triple store errored - * mid-read/write (e.g. an oxigraph worker mid-restart throwing - * `store is closed`) or its live signer-registration chain lookup threw + * mid-read/write (e.g. a managed or external store restarting and refusing + * a connection) or its live signer-registration chain lookup threw * (a degraded shared RPC). Introduced after the testnet storage-ACK * dead-air incident: every such failure previously THREW out of the * handler, which ProtocolRouter's inbound wrapper surfaces as a bare @@ -151,7 +151,7 @@ export const TRANSIENT_STORAGE_ACK_DECLINE_CODES: ReadonlySet = new Set< // waiting fixes it. STORAGE_ACK_DECLINE_CODES.MISSING_CIPHERTEXT_CHUNKS, // Dead-air fix: a store/RPC blip on the core clears when the local - // oxigraph worker or the shared RPC recovers — the same "wait a few + // local store or the shared RPC recovers — the same "wait a few // seconds and re-ask" cadence as SWM catch-up. Marking it transient // keeps a briefly-degraded core in the quorum pool instead of // deselecting it on the first blip. diff --git a/packages/core/test/ensure-dkg-node-config.test.ts b/packages/core/test/ensure-dkg-node-config.test.ts index 6c2d519fea..8fc412231c 100644 --- a/packages/core/test/ensure-dkg-node-config.test.ts +++ b/packages/core/test/ensure-dkg-node-config.test.ts @@ -71,7 +71,7 @@ describe('ensureDkgNodeConfig — store-backend default (issue #960)', () => { it('does NOT flip an existing (block-less) node onto a new backend', () => { // Simulate an existing node: a config.json is already on disk (it had been - // running on the oxigraph-worker runtime fallback). Re-running setup must + // running without an explicit store block). Re-running setup must // not silently switch its backend (which would force a store reset). writeFileSync(join(tempHome, 'config.json'), JSON.stringify({ name: 'node-a', nodeRole: 'edge' }) + '\n'); ensureDkgNodeConfig({ diff --git a/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts b/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts index dd08785081..3abebdcd3f 100644 --- a/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts +++ b/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts @@ -124,7 +124,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: true }, store: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(home, 'store.nq') }, }, chain: { diff --git a/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts index c057c13d7c..3732dc9d76 100644 --- a/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts @@ -17,7 +17,7 @@ * slow-RPC fault legs those PRs also fixed cannot be induced against a healthy * black-box devnet — they are pinned by the unit/integration suites shipped in * #1404 (publisher-runner-ack-readiness, policy-retry) and #1408 - * (storage-ack-core-unavailable, oxigraph-worker-respawn). + * (storage-ack-core-unavailable and related store-outage scenarios). */ import { test, expect } from '../../fixtures/base.js'; import { requireDevnetNode, requireDevnetPrecondition, waitForDevnetStatus } from '../../helpers/devnet.js'; diff --git a/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts index 487cdaaac2..0eb25f50bb 100644 --- a/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts @@ -10,9 +10,9 @@ * node does not track is NEVER admitted; refusals are structured, side-effect * free, and leave the node healthy" — is exactly what this spec pins against * the healthy devnet. (The outage-only legs — 503 fail-closed, the on-chain - * public rescue itself, oxigraph worker respawn — need a killed store and are + * public rescue itself and store restart recovery — need a killed store and are * pinned by #1408's real-component suites: write-preflight-resilience, - * storage-ack-core-unavailable, oxigraph-worker-respawn.) + * storage-ack-core-unavailable and related store-outage scenarios.) */ import { test, expect } from '../../fixtures/base.js'; import { devnetApiFetch, requireDevnetNode, requireDevnetPrecondition, waitForDevnetStatus } from '../../helpers/devnet.js'; diff --git a/packages/query/src/query-handler.ts b/packages/query/src/query-handler.ts index 17fd97c2c0..4bc94d6a41 100644 --- a/packages/query/src/query-handler.ts +++ b/packages/query/src/query-handler.ts @@ -405,7 +405,7 @@ export class QueryHandler { // post-materialization `.slice()`, so a high-cardinality query is fully // materialized before being truncated. Genuinely enforcing either bound // requires threading an `AbortSignal` + result cap through - // `QueryEngine.query()` → storage → the oxigraph worker. Injecting a + // `QueryEngine.query()` → the selected storage adapter. Injecting a // `LIMIT` into the user query here is NOT a safe shortcut: for scoped // queries it would push the statement into the multi-graph // solution-set-modifier rejection path (see #789), so it is diff --git a/packages/storage/README.md b/packages/storage/README.md index 698e3c2a05..08c2263bf4 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -6,7 +6,6 @@ Triple store abstraction layer for DKG V10. Provides a unified API over multiple - **Backend adapters** — pluggable triple store implementations: - `OxigraphStore` — embedded WASM/native store, no external dependencies - - `OxigraphWorkerStore` — worker-thread variant; keeps the daemon event loop free, with a per-read-operation timeout (see below) - `BlazegraphStore` — connects to a running Blazegraph SPARQL endpoint - `SparqlHttpStore` — generic adapter for any SPARQL 1.1 compliant endpoint - **Graph manager** — named graph lifecycle (create, drop, list) with contextGraph-scoped data and metadata graphs @@ -33,39 +32,6 @@ await store.insert(quads); const result = await store.query('SELECT * WHERE { ?s ?p ?o } LIMIT 10'); ``` -## Embedded worker store (`oxigraph-worker`) tuning - -The embedded worker runs **all** store operations on a single worker thread, so -a long-running or stuck op (a huge import, an expensive query) blocks every -other store-backed request behind it. Under real load this surfaces as the -daemon's `/api/status` staying green while `/api/query`, -`/api/context-graph/list`, and `/api/assertion/create` hang. A `store.options` -knob bounds that blast radius: - -| Option | Default | Purpose | -|---|---|---| -| `operationTimeoutMs` | `120000` | Reject a **read-only** op (`query`, `hasGraph`, `listGraphs`, `countQuads`) that exceeds this instead of hanging forever — that's where the user-visible hang shows up. `0` disables (restores unbounded behaviour). `close` is exempt — its final flush always runs to completion so shutdown can't drop pending writes. | - -Mutations (`insert`, `delete`, …) are intentionally **not** bounded by this -timeout. The bound only drops the *caller's* promise — the single worker thread -keeps running the op — so a "timed-out" write could still commit afterwards, -and the rest of the codebase treats a rejected `insert`/`delete` as a clean -failure. Bounding only reads surfaces a wedged worker on the paths that hang -without inventing an indeterminate write outcome. `insert()` therefore stays -strictly atomic (all quads commit or the call fails), which callers rely on. - -```jsonc -// ~/.dkg/config.json -"store": { - "backend": "oxigraph-worker", - "options": { "operationTimeoutMs": 120000 } -} -``` - -For heavy / production workloads, prefer an out-of-process SPARQL server -(`sparql-http` or `blazegraph`), which handles reads and writes concurrently -and keeps the daemon responsive under load. - ## Internal Dependencies - `@origintrail-official/dkg-core` — configuration types, logging, constants diff --git a/packages/storage/src/adapters/oxigraph-worker-impl.ts b/packages/storage/src/adapters/oxigraph-worker-impl.ts deleted file mode 100644 index 6bfce261ce..0000000000 --- a/packages/storage/src/adapters/oxigraph-worker-impl.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { parentPort, workerData } from 'node:worker_threads'; -import { OxigraphStore } from './oxigraph.js'; - -const store = new OxigraphStore(workerData?.persistPath); - -parentPort!.on('message', async (msg: { id: number; method: string; args: unknown[] }) => { - try { - const fn = (store as any)[msg.method]; - if (typeof fn !== 'function') { - parentPort!.postMessage({ id: msg.id, error: `Unknown method: ${msg.method}` }); - return; - } - const result = await fn.apply(store, msg.args); - parentPort!.postMessage({ id: msg.id, result }); - } catch (err) { - parentPort!.postMessage({ id: msg.id, error: err instanceof Error ? err.message : String(err) }); - } -}); diff --git a/packages/storage/src/adapters/oxigraph-worker.ts b/packages/storage/src/adapters/oxigraph-worker.ts deleted file mode 100644 index 110866bc84..0000000000 --- a/packages/storage/src/adapters/oxigraph-worker.ts +++ /dev/null @@ -1,737 +0,0 @@ -import { Worker } from 'node:worker_threads'; -import { existsSync } from 'node:fs'; -import { sep } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import type { TripleStore, Quad, TripleStoreQueryOptions, QueryResult, UpdateOptions } from '../triple-store.js'; -import { registerTripleStoreAdapter } from '../triple-store.js'; -import { GraphWriteGenTracker } from '../graph-write-gen.js'; - -/** - * Default per-operation timeout for the embedded worker store. The worker is - * a SINGLE thread that processes store ops FIFO, so one slow / wedged op (a - * huge import, an expensive query, or a genuinely hung worker) blocks every - * other store-backed request queued behind it. Without a bound, the caller — - * an API route, the publisher, gossip ingest — waits FOREVER. That is the - * exact signature behind issues #997 / #999 / #1002 / #1005 / #1008: - * `/api/status` (no store) stays green while `/api/query`, - * `/api/context-graph/list`, `/api/assertion/create` never return. - * - * A bounded wait turns an indefinite hang into a surfaced error the operator - * can act on (the message points at the real fix: use an external SPARQL - * server for heavy workloads). 120s is generous enough not to trip normal - * operations yet finite. Set `operationTimeoutMs: 0` to restore the old - * unbounded behaviour. - * - * IMPORTANT — the timeout is applied to READ-ONLY ops only (see - * READ_ONLY_METHODS). A read is side-effect-free, so bounding the caller's wait - * and rejecting is always a clean, determinate failure. A mutation is NOT - * bounded: the timeout only drops the caller's promise while the single worker - * thread keeps running the op, so a "timed-out" insert/delete could STILL - * commit afterwards — and the rest of the codebase treats a rejected - * insert/delete as a clean failure, which would leave partial state visible and - * retries ambiguous. The user-visible hang in the issues above is on the read - * paths (`/api/query`, `/api/context-graph/list`); bounding reads surfaces a - * wedged worker there without inventing an indeterminate mutation outcome. - */ -const DEFAULT_OPERATION_TIMEOUT_MS = 120_000; - -/** - * Backoff (ms) before each consecutive respawn attempt after an UNEXPECTED - * worker exit: the first attempt is immediate (a one-off OOM/crash should - * recover with zero visible downtime), then 1s / 5s / 30s, capped at the last - * tier. The cap keeps a persistently-crashing worker (corrupt state, chronic - * OOM on load) from melting the node in a hot spawn/crash loop while still - * retrying often enough that a transient cause self-heals. - */ -const RESPAWN_BACKOFF_MS = [0, 1_000, 5_000, 30_000]; - -/** - * Give-up bound: after this many CONSECUTIVE respawned workers die without - * serving a single successful op, stop respawning and latch the store closed - * (the pre-recovery behaviour) with fatal operator guidance. The counter - * resets on the first successful reply from a worker (see the message handler - * in spawnWorker), so occasional crashes days apart never accumulate here — - * only a genuine crash loop trips it. - */ -const MAX_CONSECUTIVE_RESPAWNS = 5; - -/** Unref'd sleep — a respawn backoff timer must not keep the process alive on its own. */ -function sleep(ms: number): Promise { - return new Promise((resolve) => { - const t = setTimeout(resolve, ms); - if (typeof t.unref === 'function') t.unref(); - }); -} - -export interface OxigraphWorkerStoreOptions { - /** - * Per-operation timeout in milliseconds for READ-ONLY ops. Default 120_000. - * 0 disables it. Mutations are intentionally never bounded (see - * DEFAULT_OPERATION_TIMEOUT_MS) so a timed-out write can't be reported as a - * clean failure while it is still in flight. - */ - operationTimeoutMs?: number; -} - -/** - * Accept only a finite, non-negative override; otherwise fall back. The result - * is floored to an INTEGER — the timeout is a millisecond count, so a fractional - * value is meaningless noise. - */ -function normalizeNonNegativeInt(value: number | undefined, fallback: number): number { - return typeof value === 'number' && Number.isFinite(value) && value >= 0 - ? Math.floor(value) - : fallback; -} - -function asAbortError(reason: unknown): Error { - return reason instanceof Error ? reason : new Error(String(reason ?? 'aborted')); -} - -/** - * Side-effect-free read methods. ONLY these are bounded by the per-op timeout: - * rejecting a read after the bound is a clean, determinate failure (nothing was - * written), so it safely surfaces a wedged worker on the exact paths that hang - * in production. Every other method mutates persisted state and is left - * unbounded — see DEFAULT_OPERATION_TIMEOUT_MS for why timing out a mutation - * would be unsafe with the current call sites. - */ -const READ_ONLY_METHODS = new Set([ - 'query', 'hasGraph', 'listGraphs', 'countQuads', -]); - -/** Rejection raised when a read op exceeds its per-op timeout. */ -export interface OxigraphWorkerTimeoutError extends Error { - code: 'OXIGRAPH_WORKER_OP_TIMEOUT'; - /** Which store method timed out. */ - method: string; - /** The bound that was exceeded. */ - timeoutMs: number; -} - -/** - * Explicit worker POLICY state, replacing the old cluster of interdependent - * booleans/promises that all had to agree. `respawnGaveUp` folds into the - * 'gave_up' state and the new terminal 'in_memory_lost' state; `workerExited` - * stays as a separate lower-level FACT (it is orthogonal — see the field - * comment), and `closePromise`/`respawnPromise`/`consecutiveRespawnFailures` - * still carry their own orthogonal data. A single discriminated field makes the - * invalid states that used to be representable — e.g. "gave up" AND "live", or - * an in-memory store silently marked healthy after a crash — impossible to - * construct, and it gives every read a single, obvious source of truth. The - * state graph is: - * - * live ──unexpected exit, persisted──▶ respawning ──spawned──▶ live - * live ──unexpected exit, in-memory──▶ in_memory_lost (terminal — finding 1) - * respawning ──crash-loop bound hit──▶ gave_up (terminal) - * live | respawning ──close()────────▶ closing ──drained──▶ closed (terminal) - * - * `respawning` is the only state in which the current worker thread is dead but - * a replacement is on the way, so parked ops wait it out (see callAfterRespawn). - * Every other non-`live` state means "no usable worker" and fails ops fast. - */ -type WorkerLifecycle = - | 'initializing' - | 'live' - | 'respawning' - | 'closing' - | 'closed' - | 'gave_up' - | 'in_memory_lost'; - -/** Terminal states: the store will never serve another op and never respawns. */ -const TERMINAL: ReadonlySet = new Set([ - 'closed', 'gave_up', 'in_memory_lost', -]); - -export class OxigraphWorkerStore implements TripleStore { - readonly queryCancellation = 'interruptible' as const; - - // Assigned by spawnWorker(), which the constructor always calls — hence the - // definite-assignment assertion instead of an initializer. - private worker!: Worker; - private nextId = 0; - private pending = new Map void; reject: (e: Error) => void }>(); - // #1609: per-graph write generations, bumped client-side after each - // successful mutation RPC (the worker owns no caller-visible caches). - // Feeds the chain-reconcile negative memo via `asGraphWriteGenSource`. - private readonly writeGen = new GraphWriteGenTracker(); - private readonly operationTimeoutMs: number; - /** Resolved path of the compiled worker impl; reused verbatim on respawn. */ - private readonly workerPath: string; - /** Persistence file handed to every spawned worker (undefined = in-memory). */ - private readonly persistPath: string | undefined; - /** - * Single source of truth for the store's high-level POLICY state (see - * WorkerLifecycle). It subsumes the former `respawnGaveUp` boolean (⇔ state - * === 'gave_up') and adds the terminal 'in_memory_lost' state that finding-1 - * needs, so the give-up/close/data-lost verdicts can never disagree the way - * an ad-hoc cluster of interdependent booleans could. Set to 'live' the - * moment spawnWorker() arms a fresh thread; only ever moves along the - * documented transitions (every write is guarded — `setLifecycle` for the - * non-spawn hops and `markSpawnedLive` for the spawn → 'live' hop — so a - * terminal state can never silently reopen). - * - * `workerExited` below stays as a separate, lower-level FACT ("the current - * thread has exited") because it is genuinely orthogonal to policy: during a - * graceful close() the state is 'closing' while the thread is still alive and - * mid-flush, then the thread exits — same policy state, different fact. The - * old soup was the *interdependence* of workerExited + respawnGaveUp + - * respawnPromise + closePromise all having to agree; folding the policy half - * into one discriminated field removes that. - * - * The initializer is a pre-construction placeholder only: the constructor - * always calls spawnWorker() (whose `markSpawnedLive` accepts the - * non-terminal 'initializing' placeholder), so no op ever observes this - * value. 'initializing' is deliberately NON-terminal so the spawn guard can - * enforce terminal permanence uniformly — a placeholder of 'closed' (a - * terminal state) would have made the first legal spawn indistinguishable - * from illegally reopening a closed store. - */ - private lifecycle: WorkerLifecycle = 'initializing'; - /** Set once the CURRENT worker thread has exited (graceful close, crash, or kill); cleared by spawnWorker(). */ - private workerExited = false; - /** Memoized close so repeat/concurrent close() calls share one teardown. */ - private closePromise: Promise | null = null; - /** - * In-flight replacement of a crashed worker. While set, NEW ops park on it - * (see callWithTimeout) instead of failing, so a request that races the - * respawn sees a slightly slower success rather than a spurious error. - * Null whenever a live worker is armed. Tracked separately from `lifecycle` - * because ops need the actual promise to await, not just the 'respawning' tag. - */ - private respawnPromise: Promise | null = null; - /** - * How many respawned workers in a row died without answering a single op - * successfully. Reset to 0 by the first successful reply; feeds the backoff - * tier and the MAX_CONSECUTIVE_RESPAWNS give-up bound. - */ - private consecutiveRespawnFailures = 0; - - constructor(persistPath?: string, opts?: OxigraphWorkerStoreOptions) { - this.operationTimeoutMs = normalizeNonNegativeInt(opts?.operationTimeoutMs, DEFAULT_OPERATION_TIMEOUT_MS); - - // Resolve the worker impl with a small search path so this keeps - // working in all three deployment shapes we actually run in: - // - // 1. Production / npm install / built monorepo — this module is - // loaded from `dist/adapters/oxigraph-worker.js`, so the - // sibling `./oxigraph-worker-impl.js` resolves correctly. - // 2. vitest against raw source — this module is loaded from - // `src/adapters/oxigraph-worker.ts`, so the sibling - // `./oxigraph-worker-impl.js` does NOT exist, but its compiled - // twin in `dist/adapters/` does as long as the caller ran - // `pnpm --filter ...dkg-storage build` first. Redirect to - // that path so the adapter is runnable in dev loops. - // 3. Neither file exists — genuinely unbuilt tree. Throw a loud, - // actionable error explaining the fix (`pnpm build`), matching - // the expectation in `test/storage.test.ts`. - const siblingJsUrl = new URL('./oxigraph-worker-impl.js', import.meta.url); - const siblingJsPath = fileURLToPath(siblingJsUrl); - let workerPath: string | null = existsSync(siblingJsPath) ? siblingJsPath : null; - if (!workerPath) { - const srcAdapters = `${sep}src${sep}adapters${sep}`; - const distAdapters = `${sep}dist${sep}adapters${sep}`; - if (siblingJsPath.includes(srcAdapters)) { - const candidate = siblingJsPath.replace(srcAdapters, distAdapters); - if (existsSync(candidate)) workerPath = candidate; - } - } - if (!workerPath) { - throw new Error( - `oxigraph-worker adapter: compiled worker artefact ` + - `\`oxigraph-worker-impl.js\` was not found next to ` + - `${siblingJsPath} or in the sibling \`dist/adapters/\` ` + - `directory. Run \`pnpm --filter @origintrail-official/dkg-storage build\` ` + - `before using this adapter.`, - ); - } - this.workerPath = workerPath; - this.persistPath = persistPath; - this.spawnWorker(); - } - - /** - * The single writer for `this.lifecycle`. A typed setter (rather than raw - * field assignment scattered across the class) gives us one place to assert - * the invariant that matters: a TERMINAL state is a dead end. Without this - * guard a latent bug — a stray respawn resurrecting a `gave_up` store, or a - * crash handler firing after `close()` — could silently re-open a store the - * operator was told is gone, which is exactly the class of "healthy-looking - * but wrong" state finding 1 is about. If we ever try to leave a terminal - * state we throw loudly instead. - */ - private setLifecycle(next: WorkerLifecycle): void { - if (this.lifecycle === next) return; - if (TERMINAL.has(this.lifecycle)) { - throw new Error( - `oxigraph-worker: illegal lifecycle transition ${this.lifecycle} → ${next} ` + - '(terminal states are permanent) — this is a bug in the respawn supervisor.', - ); - } - this.lifecycle = next; - } - - /** - * The ONE guarded transition INTO 'live' — used by spawnWorker() for both the - * constructor's first spawn (from the 'initializing' placeholder) and a - * respawn (from 'respawning'). `setLifecycle` can't own this hop because - * entering 'live' is legal only from those two non-terminal states, but the - * terminal-permanence invariant must still hold: a spawn attempted after - * 'closed' / 'gave_up' / 'in_memory_lost' (e.g. a future respawn/close code - * path calling spawnWorker() by mistake) MUST throw rather than silently - * resurrect a store that promised it would never serve another op. - */ - private markSpawnedLive(): void { - if (this.lifecycle === 'live') return; - if (this.lifecycle !== 'initializing' && this.lifecycle !== 'respawning') { - throw new Error( - `oxigraph-worker: illegal spawn transition ${this.lifecycle} → live ` + - '(a worker may only be spawned from initializing or respawning; ' + - 'terminal states are permanent) — this is a bug in the respawn supervisor.', - ); - } - this.lifecycle = 'live'; - } - - /** True once an operator-initiated close() has begun (closing or closed). */ - private get isClosing(): boolean { - return this.lifecycle === 'closing' || this.lifecycle === 'closed'; - } - - /** - * Create the worker thread and arm its lifecycle handlers. Shared by the - * constructor and by respawn() so a replacement worker is wired IDENTICALLY - * to the original — same impl path, same workerData, same handlers — and the - * OxigraphWorkerStore object keeps its identity, so every alias held across - * the daemon (routes, publisher, gossip ingest) transparently talks to the - * new thread. Each handler captures the worker it was armed for and no-ops - * if `this.worker` has since moved on, so a stale event from an - * already-replaced thread can never clobber the live one's state. - */ - private spawnWorker(): void { - const worker = new Worker(this.workerPath, { - workerData: { persistPath: this.persistPath }, - }); - this.worker = worker; - this.workerExited = false; - // Arming a fresh thread IS the transition into 'live'. Route it through the - // guarded `markSpawnedLive` (not `setLifecycle`, which forbids ALL entries - // to 'live'): it permits the two legal predecessors — the 'initializing' - // placeholder (constructor's first spawn) and 'respawning' (a respawn) — - // while still throwing on terminal states, so a stray spawn after close / - // give-up / in-memory-loss can never silently resurrect the store. close() - // and the respawn supervisor are the only other writers and never race - // this: a spawn only happens in the constructor or within respawn(), which - // bails the moment a close() is seen. - this.markSpawnedLive(); - worker.on('message', (msg: { id: number; result?: unknown; error?: string }) => { - if (this.worker !== worker) return; - // Any successful reply proves this worker is healthy, which ends the - // crash-loop accounting window (see MAX_CONSECUTIVE_RESPAWNS). - if (!msg.error) this.consecutiveRespawnFailures = 0; - const p = this.pending.get(msg.id); - if (!p) return; - this.pending.delete(msg.id); - if (msg.error) p.reject(new Error(msg.error)); - else p.resolve(msg.result); - }); - worker.on('error', (err) => { - if (this.worker !== worker) return; - // Loud by design. 'error' means the worker thread threw an uncaught - // exception (an 'exit' event follows). This used to be silent, which is - // how a testnet node served HTTP for DAYS with a dead store — green - // /api/status, 503 on every write — and nothing in the logs said why. - console.error('[oxigraph-worker] worker thread error (thread is going down):', err); - for (const [, p] of this.pending) p.reject(err); - this.pending.clear(); - }); - // Once the worker exits, no pending op can ever get a reply. Settle them all - // (reject) instead of leaving callers hung forever — this is what makes the - // unbounded `close()` safe: if a second close (or any op) is still queued - // when terminate() kills the thread, it rejects here rather than hanging. - // - // Three very different exits land here (branch on lifecycle + persistPath): - // • intentional — close() ran (state is 'closing'). Stay closed; close() - // owns the final transition to 'closed'. - // • unexpected, disk-persisted — the thread died on its own. - // ERR_WORKER_OUT_OF_MEMORY, for example, kills ONLY the worker thread, - // not the process, so the daemon would otherwise keep serving with a - // permanently dead store. The committed state is on disk, so a fresh - // worker on the same path reopens it — go 'respawning' and recover. - // • unexpected, IN-MEMORY — there is no disk to reload from, so a fresh - // worker would come up EMPTY and look perfectly healthy while every - // row written before the crash is silently gone (confirmed live: data - // vanished). That is unrecoverable, so we FAIL CLOSED into the terminal - // 'in_memory_lost' state instead of respawning an empty store — every - // later op then rejects with a clear data-loss error (see postToWorker) - // rather than returning wrong-but-plausible results. - worker.on('exit', (code) => { - if (this.worker !== worker) return; - this.workerExited = true; - const intentional = this.isClosing; - // Distinguish the two unexpected exits up front so the log and the - // lifecycle transition below agree on what just happened. - const inMemoryLost = !intentional && this.persistPath === undefined; - if (intentional) { - console.info(`[oxigraph-worker] worker exited (code ${code}) — initiated by close()`); - } else if (inMemoryLost) { - console.error( - `[oxigraph-worker] worker exited UNEXPECTEDLY (code ${code}) — nobody called close(). ` + - 'FATAL: this store is IN-MEMORY (no persistPath), so its entire contents were lost with the ' + - 'worker thread and CANNOT be recovered. Failing the store closed instead of silently continuing ' + - 'with an empty store — every store-backed request will now fail fast. Use a disk-persisted store ' + - '(store.path) if the workload must survive a worker crash.', - ); - } else { - console.error( - `[oxigraph-worker] worker exited UNEXPECTEDLY (code ${code}) — nobody called close(). ` + - 'The store is disk-persisted; respawning a fresh worker on the same path.', - ); - } - if (this.pending.size > 0) { - const err = intentional - ? new Error('oxigraph-worker: worker exited before the operation completed (store closed)') - : inMemoryLost - ? new Error( - `oxigraph-worker: the IN-MEMORY worker exited unexpectedly (code ${code}) before the operation ` + - 'completed and its data was lost — an in-memory store cannot be recovered from a worker crash.', - ) - : new Error( - `oxigraph-worker restarted — retry: the worker exited unexpectedly (code ${code}) before the ` + - 'operation completed, so its outcome is unknown; a replacement worker is being spawned.', - ); - for (const [, p] of this.pending) p.reject(err); - this.pending.clear(); - } - // Route the state transition. close() already moved us to 'closing' and - // owns the final hop to 'closed', so the intentional case touches nothing. - if (inMemoryLost) { - this.setLifecycle('in_memory_lost'); - } else if (!intentional) { - this.scheduleRespawn(); - } - }); - } - - /** Arm (at most one) background respawn; the policy lives in respawn(). */ - private scheduleRespawn(): void { - // Never resurrect a store that has already given up, been closed, or lost - // its in-memory data, and never stack a second supervisor on an in-flight - // one. This is only ever reached from the exit handler's disk-persisted - // unexpected-exit branch, so we're normally transitioning out of 'live'. - if (this.respawnPromise || TERMINAL.has(this.lifecycle)) return; - this.setLifecycle('respawning'); - // respawn() never rejects (every failure path is caught and looped or - // latched), so this promise can't become an unhandled rejection while no - // op happens to be parked on it. - this.respawnPromise = this.respawn().finally(() => { - this.respawnPromise = null; - }); - } - - /** - * Replace a crashed worker with a fresh one: immediate first attempt, then - * capped backoff (RESPAWN_BACKOFF_MS) between consecutive attempts, giving - * up for good after MAX_CONSECUTIVE_RESPAWNS workers in a row die without - * serving a single successful op. Giving up latches the store closed — - * exactly the pre-recovery behaviour — but with a FATAL log telling the - * operator what happened and where to look, instead of the old silence. - */ - private async respawn(): Promise { - while (true) { - // close() raced the respawn — honour it. An operator-initiated close - // must never be resurrected by the supervisor. (close() flips lifecycle - // to 'closing' atomically before awaiting, so isClosing is the signal.) - if (this.isClosing) return; - if (this.consecutiveRespawnFailures >= MAX_CONSECUTIVE_RESPAWNS) { - this.setLifecycle('gave_up'); - console.error( - `[oxigraph-worker] FATAL: the worker died ${MAX_CONSECUTIVE_RESPAWNS} times in a row without ` + - 'serving a single successful operation — giving up on respawn and latching the store closed. ' + - 'Every store-backed request will now fail fast. Investigate the crash cause (worker OOM → raise ' + - 'memory or move to an external SPARQL backend via store.backend "sparql-http" / "blazegraph"; ' + - 'corrupt persist file → see the quarantine log) and restart the node.', - ); - return; - } - const attempt = this.consecutiveRespawnFailures; - this.consecutiveRespawnFailures += 1; - const delayMs = RESPAWN_BACKOFF_MS[Math.min(attempt, RESPAWN_BACKOFF_MS.length - 1)]; - if (delayMs > 0) { - console.error( - `[oxigraph-worker] respawn attempt ${attempt + 1}/${MAX_CONSECUTIVE_RESPAWNS} in ${delayMs}ms…`, - ); - await sleep(delayMs); - // Re-check: close() may have arrived during the backoff sleep. - if (this.isClosing) return; - } - try { - this.spawnWorker(); - console.info(`[oxigraph-worker] respawned worker (attempt ${attempt + 1}/${MAX_CONSECUTIVE_RESPAWNS})`); - return; - } catch (err) { - // new Worker() itself can throw (e.g. the impl file vanished at - // runtime). Treat it exactly like a worker that died instantly and - // loop into the next backoff tier. - console.error('[oxigraph-worker] respawn attempt failed to start a worker:', err); - } - } - } - - private call(method: string, ...args: unknown[]): Promise { - // Only read-only ops are bounded; mutations run unbounded (timeoutMs 0) so a - // timed-out write is never reported as a clean failure while still in flight. - const timeoutMs = READ_ONLY_METHODS.has(method) ? this.operationTimeoutMs : 0; - return this.callWithTimeout(timeoutMs, undefined, method, ...args); - } - - /** - * Post one op to the (live) worker and await its reply, bounding the caller's - * wait by `timeoutMs` (0 = wait indefinitely). The bound is per-CALLER: on - * timeout or caller abort we reject and drop the pending entry, but the - * single-threaded worker is STILL running the op — the late reply is then - * ignored (the message handler no-ops on a missing id) rather than - * double-settling this promise. Only read-only ops are ever given a non-zero - * timeout (see `call`), so a fired timeout is always a determinate, - * side-effect-free failure. If a crashed worker is being replaced, the op - * first parks on the respawn (the timeout starts only once it is actually - * posted — backoff is capped well below the default read bound anyway). - */ - private callWithTimeout( - timeoutMs: number, - signal: AbortSignal | undefined, - method: string, - ...args: unknown[] - ): Promise { - // A crashed worker may be mid-replacement right now. Park the op on the - // respawn instead of failing it: the store is disk-persisted and about to - // come back, so the caller sees a slightly slower success rather than a - // spurious "store is closed" for a condition the store is already fixing. - if (this.respawnPromise) return this.callAfterRespawn(timeoutMs, signal, method, args); - return this.postToWorker(timeoutMs, signal, method, args); - } - - /** - * Wait out the in-flight respawn(s), then post as normal. A loop rather - * than a single await because the replacement worker can itself die before - * this op gets posted, arming a new respawnPromise. If the respawn gave up, - * postToWorker's workerExited guard turns this into the fail-fast closed - * error (with the crash-loop guidance appended). - */ - private async callAfterRespawn( - timeoutMs: number, - signal: AbortSignal | undefined, - method: string, - args: unknown[], - ): Promise { - while (this.respawnPromise) await this.respawnPromise; - return this.postToWorker(timeoutMs, signal, method, args); - } - - private postToWorker( - timeoutMs: number, - signal: AbortSignal | undefined, - method: string, - args: unknown[], - ): Promise { - const id = this.nextId++; - return new Promise((resolve, reject) => { - // The worker is gone (closed, crashed with respawn given up, or an - // in-memory store whose data was lost) — a posted message would never be - // answered, so fail fast instead of registering a pending entry that can - // only ever hang. The lifecycle picks the RIGHT diagnostic: - // • in_memory_lost — the data is unrecoverable; say so plainly instead - // of pretending the store merely "closed" (finding 1: never let an - // in-memory crash look like a clean close or an empty-but-healthy store). - // • gave_up — closed + the crash-loop guidance. - // • otherwise — a plain operator close. - if (this.workerExited) { - if (this.lifecycle === 'in_memory_lost') { - reject(new Error( - `oxigraph-worker: cannot run "${method}" — the IN-MEMORY store's worker crashed and its data was ` + - 'lost. An in-memory store cannot be recovered from a worker crash; every request now fails ' + - 'fast. Use a disk-persisted store (store.path) or restart the node to start from empty.', - )); - return; - } - reject(new Error( - `oxigraph-worker: cannot run "${method}" — the store is closed.` + - (this.lifecycle === 'gave_up' - ? ' (The worker crashed repeatedly and automatic respawn gave up — restart the node and ' + - 'investigate the [oxigraph-worker] crash logs.)' - : ''), - )); - return; - } - if (signal?.aborted) { - reject(asAbortError(signal.reason)); - return; - } - let timer: ReturnType | undefined; - let onAbort: (() => void) | undefined; - const cleanup = () => { - if (timer) clearTimeout(timer); - if (signal && onAbort) signal.removeEventListener('abort', onAbort); - }; - if (timeoutMs > 0) { - timer = setTimeout(() => { - if (this.pending.delete(id)) { - cleanup(); - const err = new Error( - `oxigraph-worker: "${method}" timed out after ${timeoutMs}ms. ` + - `The embedded store runs on a single worker thread, so a long-running or ` + - `stuck operation blocks all reads queued behind it. For heavy workloads ` + - `point the node at an external SPARQL server (store.backend "sparql-http" ` + - `/ "blazegraph"), or raise / disable store.options.operationTimeoutMs.`, - ) as OxigraphWorkerTimeoutError; - err.code = 'OXIGRAPH_WORKER_OP_TIMEOUT'; - err.method = method; - err.timeoutMs = timeoutMs; - reject(err); - } - }, timeoutMs); - // A pending-op timer must not keep the process alive on its own. - if (typeof timer.unref === 'function') timer.unref(); - } - this.pending.set(id, { - resolve: (v) => { cleanup(); resolve(v as T); }, - reject: (e) => { cleanup(); reject(e); }, - }); - if (signal) { - onAbort = () => { - if (this.pending.delete(id)) { - cleanup(); - reject(asAbortError(signal.reason)); - } - }; - signal.addEventListener('abort', onAbort, { once: true }); - if (signal.aborted) { - onAbort(); - return; - } - } - try { - this.worker.postMessage({ id, method, args }); - } catch (err) { - if (this.pending.delete(id)) { - cleanup(); - reject(err instanceof Error ? err : new Error(String(err))); - } - } - }); - } - - // A single atomic worker message: all quads commit together or the call - // fails. The contract is "all-or-nothing" and every caller can rely on it - // (e.g. FinalizationHandler packs both canonical copies into one insert so one - // can't land without the other). Large idempotent bulk imports that want - // head-of-line fairness should run on an external SPARQL server, not by - // silently fragmenting this insert into non-atomic chunks. - async insert(quads: Quad[]): Promise { - await this.call('insert', quads); - this.writeGen.recordGraphWrites(new Set(quads.map((q) => q.graph || ''))); - } - async delete(quads: Quad[]): Promise { - await this.call('delete', quads); - this.writeGen.recordGraphWrites(new Set(quads.map((q) => q.graph || ''))); - } - async deleteByPattern(pattern: Partial): Promise { - const removed = await this.call('deleteByPattern', pattern); - if (pattern.graph) this.writeGen.recordGraphWrites([pattern.graph]); - else this.writeGen.recordUnscopedWrite(); - return removed; - } - // Server-side SPARQL UPDATE forwarded to the worker's OxigraphStore (which - // implements `update`); same atomic single-message contract as `insert`. - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async update(sparql: string, _options?: UpdateOptions): Promise { - await this.call('update', sparql); - // A raw UPDATE's write scope is not derivable at the call site - // (`touchedGraphs` hints only membership changes) — unscoped bump. - this.writeGen.recordUnscopedWrite(); - } - async query(sparql: string, options?: TripleStoreQueryOptions): Promise { - return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'query', sparql); - } - async hasGraph(graphUri: string, options?: TripleStoreQueryOptions): Promise { - return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'hasGraph', graphUri); - } - async createGraph(graphUri: string): Promise { return this.call('createGraph', graphUri); } - async dropGraph(graphUri: string): Promise { - await this.call('dropGraph', graphUri); - this.writeGen.recordGraphWrites([graphUri]); - } - async listGraphs(options?: TripleStoreQueryOptions): Promise { - return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'listGraphs'); - } - async deleteBySubjectPrefix(graphUri: string, prefix: string): Promise { - const removed = await this.call('deleteBySubjectPrefix', graphUri, prefix); - this.writeGen.recordGraphWrites([graphUri]); - return removed; - } - - /** {@link GraphWriteGenSource} capability (#1609) — see graph-write-gen.ts. */ - getWriteGen(graphPrefix: string): number { - return this.writeGen.getWriteGen(graphPrefix); - } - async countQuads(graphUri?: string): Promise { return this.call('countQuads', graphUri); } - async flush(): Promise { return this.call('flush'); } - - async close(): Promise { - // Memoized + serialized: every close() call shares ONE teardown promise, so - // we issue exactly one close RPC. (A second unbounded close RPC would be - // orphaned when terminate() kills the worker after the first resolves, and - // — without the exit handler — would hang forever.) - // - // Flip the lifecycle to 'closing' FIRST (before awaiting anything), unless - // the store already reached a terminal state on its own (gave_up / - // in_memory_lost, or an even earlier close). Doing it synchronously here is - // what lets an in-flight respawn see the close and bail (respawn() checks - // isClosing) and what makes the worker's 'exit' handler treat this exit as - // intentional. A store that already latched terminal is simply reaped below - // — its state is permanent, so we must NOT try to move it to 'closing'. - if (!this.closePromise) { - if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closing'); - this.closePromise = this.doClose(); - } - return this.closePromise; - } - - private async doClose(): Promise { - // Worker already gone (crash/kill, respawn give-up, or in-memory loss) — - // there's nothing to flush; just make sure the thread is reaped. Keeps - // close() idempotent and non-throwing. Only settle into the terminal - // 'closed' state when we're not already in a *different* terminal state - // (gave_up / in_memory_lost stay as-is so their diagnostics survive). - if (this.workerExited) { - try { await this.worker.terminate(); } catch { /* already terminated */ } - if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closed'); - return; - } - // `close` runs the worker's FINAL synchronous flush (insert() only schedules - // a 50ms debounced flush, so close is what guarantees durability). It is - // therefore EXEMPT from the per-op timeout (timeoutMs 0): bounding it could - // fire the timeout while the worker is mid-flush, and the `finally` would - // then terminate() the thread before pending writes hit disk — losing data. - // terminate() always runs in `finally`; the worker's 'exit' handler then - // rejects anything still pending, so nothing leaks or hangs. - try { - await this.callWithTimeout(0, undefined, 'close'); - } finally { - await this.worker.terminate(); - // The 'exit' handler above left us in 'closing' (intentional exit). Land - // the terminal transition here so a post-close op fails fast (workerExited - // is now set, and the guard reads 'closed' for the plain message). - if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closed'); - } - } -} - -registerTripleStoreAdapter('oxigraph-worker', async (opts) => { - const filePath = opts?.path as string | undefined; - return new OxigraphWorkerStore(filePath, { - operationTimeoutMs: - typeof opts?.operationTimeoutMs === 'number' ? (opts.operationTimeoutMs as number) : undefined, - }); -}); diff --git a/packages/storage/src/adapters/oxigraph.ts b/packages/storage/src/adapters/oxigraph.ts index f476b3176d..d5c9aa8522 100644 --- a/packages/storage/src/adapters/oxigraph.ts +++ b/packages/storage/src/adapters/oxigraph.ts @@ -254,8 +254,8 @@ export class OxigraphStore implements TripleStore { async query(sparql: string, options?: TripleStoreQueryOptions): Promise { throwIfAborted(options?.signal); // The embedded Oxigraph binding executes synchronously, so a caller abort - // cannot interrupt this native call mid-flight. Use oxigraph-worker or an - // HTTP backend when long sync queries need prompt cancellation. + // cannot interrupt this native call mid-flight. Use an HTTP backend when + // long sync queries need prompt cancellation. const result = this.store.query(sparql); throwIfAborted(options?.signal); diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 5e8f4ee97c..c916a1843f 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -10,16 +10,35 @@ export { type AskResult, type TripleStoreConfig, type TripleStoreBackend, + type TripleStoreFactoryConfig, type TripleStoreQueryOptions, type UpdateOptions, type LargeLiteralStorageConfig, registerTripleStoreAdapter, createTripleStore, + toTripleStoreBackend, tryUpdateWithTouchedGraphs, - isExternalBackend, getSparqlEndpoint, type SparqlEndpoint, + type SparqlEndpointStoreConfig, } from './triple-store.js'; +export { + STORAGE_ADAPTERS, + classifyTripleStoreBackend, + customTripleStoreBackend, + getStorageAdapterPolicy, + isExternalBackend, + isStorageAdapterBackend, + storageAdapterNames, + type ClassifiedTripleStoreBackend, + type CustomTripleStoreBackend, + type ExternalStoreBackend, + type LocalStoreBackend, + type StorageAdapterBackend, + type StorageAdapterKind, + type StorageAdapterOfKind, + type StorageAdapterPolicy, +} from './store-backends.js'; export { StorePriorityScheduler, externalStorePriorityScheduler, @@ -60,7 +79,6 @@ export { } from './graph-write-gen.js'; export { OxigraphStore } from './adapters/oxigraph.js'; -export { OxigraphWorkerStore } from './adapters/oxigraph-worker.js'; export { BlazegraphStore } from './adapters/blazegraph.js'; export { SparqlHttpStore, @@ -87,6 +105,5 @@ export { PrivateContentStore } from './private-store.js'; // Side-effect: register built-in adapters import './adapters/oxigraph.js'; -import './adapters/oxigraph-worker.js'; import './adapters/blazegraph.js'; import './adapters/sparql-http.js'; diff --git a/packages/storage/src/store-backends.ts b/packages/storage/src/store-backends.ts new file mode 100644 index 0000000000..0d07ba1b2b --- /dev/null +++ b/packages/storage/src/store-backends.ts @@ -0,0 +1,84 @@ +/** + * Canonical metadata for adapters the storage factory can construct. + * + * Daemon defaults, retired config names, migration policy, and CLI labels live + * in the CLI package. Keeping this registry adapter-only prevents the storage + * layer from acquiring daemon lifecycle or presentation policy. + */ +export const STORAGE_ADAPTERS = { + oxigraph: { + kind: 'local', + requiresExistingPath: false, + }, + 'oxigraph-persistent': { + kind: 'local', + requiresExistingPath: true, + }, + blazegraph: { + kind: 'external', + queryEndpointOption: 'url', + updateEndpointOption: 'url', + }, + 'sparql-http': { + kind: 'external', + queryEndpointOption: 'queryEndpoint', + updateEndpointOption: 'updateEndpoint', + authOption: 'auth', + }, +} as const; + +export type StorageAdapterBackend = keyof typeof STORAGE_ADAPTERS; +export type StorageAdapterPolicy = (typeof STORAGE_ADAPTERS)[StorageAdapterBackend]; +export type StorageAdapterKind = StorageAdapterPolicy['kind']; +export type StorageAdapterOfKind = { + [Backend in StorageAdapterBackend]: typeof STORAGE_ADAPTERS[Backend] extends { kind: Kind } + ? Backend + : never; +}[StorageAdapterBackend]; +export type ExternalStoreBackend = StorageAdapterOfKind<'external'>; +export type LocalStoreBackend = StorageAdapterOfKind<'local'>; + +declare const CUSTOM_TRIPLE_STORE_BACKEND: unique symbol; +export type CustomTripleStoreBackend = string & { + readonly [CUSTOM_TRIPLE_STORE_BACKEND]: true; +}; + +export type ClassifiedTripleStoreBackend = + | { kind: 'adapter'; backend: StorageAdapterBackend } + | { kind: 'custom'; backend: CustomTripleStoreBackend }; + +export function storageAdapterNames(): StorageAdapterBackend[] { + return Object.keys(STORAGE_ADAPTERS) as StorageAdapterBackend[]; +} + +export function isStorageAdapterBackend( + backend: string | undefined | null, +): backend is StorageAdapterBackend { + return backend != null && Object.prototype.hasOwnProperty.call(STORAGE_ADAPTERS, backend); +} + +export function getStorageAdapterPolicy( + backend: string | undefined | null, +): StorageAdapterPolicy | undefined { + return isStorageAdapterBackend(backend) ? STORAGE_ADAPTERS[backend] : undefined; +} + +export function isExternalBackend( + backend: string | undefined | null, +): backend is ExternalStoreBackend { + return isStorageAdapterBackend(backend) && STORAGE_ADAPTERS[backend].kind === 'external'; +} + +export function customTripleStoreBackend(backend: string): CustomTripleStoreBackend { + if (!backend.trim()) throw new Error('Custom triple-store backend name cannot be empty'); + if (isStorageAdapterBackend(backend)) { + throw new Error(`Known triple-store adapter "${backend}" does not need a custom-backend wrapper`); + } + return backend as CustomTripleStoreBackend; +} + +export function classifyTripleStoreBackend(backend: string): ClassifiedTripleStoreBackend { + return isStorageAdapterBackend(backend) + ? { kind: 'adapter', backend } + : { kind: 'custom', backend: backend as CustomTripleStoreBackend }; +} diff --git a/packages/storage/src/triple-store.ts b/packages/storage/src/triple-store.ts index 329c91e529..9ad2a779b7 100644 --- a/packages/storage/src/triple-store.ts +++ b/packages/storage/src/triple-store.ts @@ -17,6 +17,15 @@ import { ChangelogStore, type ChangelogStoreOptions, } from './changelog-store.js'; +import { + classifyTripleStoreBackend, + getStorageAdapterPolicy, + isExternalBackend, + type CustomTripleStoreBackend, + type StorageAdapterBackend, +} from './store-backends.js'; + +export { isExternalBackend } from './store-backends.js'; export interface Quad { subject: string; @@ -166,11 +175,19 @@ export async function tryUpdateWithTouchedGraphs( return true; } -export type TripleStoreBackend = 'oxigraph' | 'oxigraph-persistent' | 'oxigraph-worker' | 'blazegraph' | 'sparql-http' | string; +/** + * Backends the storage factory can construct: registered built-in adapters or + * an explicitly branded custom adapter name. + */ +export type TripleStoreBackend = StorageAdapterBackend | CustomTripleStoreBackend; + +/** Explicitly cross from stringly daemon/user config into the factory model. */ +export function toTripleStoreBackend(backend: string): TripleStoreBackend { + const classification = classifyTripleStoreBackend(backend); + return classification.backend; +} -// Backends that talk to a remote SPARQL endpoint over HTTP rather than -// owning local files. The local/external split governs three pieces of -// daemon behaviour: +// The canonical backend taxonomy governs three pieces of daemon behaviour: // 1. Store-size metric — local backends report file bytes, external // backends have no file to stat (`getStoreBytes` returns null). // 2. Chain-reset wipe — local backends `rm` files, external backends @@ -181,12 +198,6 @@ export type TripleStoreBackend = 'oxigraph' | 'oxigraph-persistent' | 'oxigraph- // 3. Boot health check — for external backends, an ASK probe runs at // daemon start; an unreachable endpoint exits the daemon with an // actionable message rather than booting half-broken. -const EXTERNAL_BACKENDS: ReadonlySet = new Set(['blazegraph', 'sparql-http']); - -export function isExternalBackend(backend: string | undefined | null): boolean { - return typeof backend === 'string' && EXTERNAL_BACKENDS.has(backend); -} - /** * Shape-normalised SPARQL endpoint extracted from a TripleStoreConfig. * @@ -202,34 +213,39 @@ export interface SparqlEndpoint { headers: Record; } -export function getSparqlEndpoint(storeConfig: TripleStoreConfig): SparqlEndpoint { +export interface SparqlEndpointStoreConfig { + backend: string; + options?: Record; +} + +export function getSparqlEndpoint(storeConfig: SparqlEndpointStoreConfig): SparqlEndpoint { if (!isExternalBackend(storeConfig.backend)) { throw new Error( `getSparqlEndpoint called for non-external backend "${storeConfig.backend}"`, ); } const opts = (storeConfig.options ?? {}) as Record; - if (storeConfig.backend === 'blazegraph') { - const url = typeof opts.url === 'string' ? opts.url : ''; - if (!url) { - throw new Error('blazegraph storeConfig requires options.url'); - } - return { queryUrl: url, updateUrl: url, headers: {} }; + const policy = getStorageAdapterPolicy(storeConfig.backend); + if (!policy || policy.kind !== 'external') { + throw new Error(`No external-store policy found for "${storeConfig.backend}"`); } - // sparql-http - const queryEndpoint = typeof opts.queryEndpoint === 'string' ? opts.queryEndpoint : ''; - if (!queryEndpoint) { - throw new Error('sparql-http storeConfig requires options.queryEndpoint'); + const queryOption = policy.queryEndpointOption; + const queryUrl = typeof opts[queryOption] === 'string' ? opts[queryOption] as string : ''; + if (!queryUrl) { + throw new Error(`${storeConfig.backend} storeConfig requires options.${queryOption}`); } - const updateEndpoint = - typeof opts.updateEndpoint === 'string' && opts.updateEndpoint - ? opts.updateEndpoint - : queryEndpoint; + const updateOption = policy.updateEndpointOption; + const updateUrl = typeof opts[updateOption] === 'string' && opts[updateOption] + ? opts[updateOption] as string + : queryUrl; const headers: Record = {}; - if (typeof opts.auth === 'string' && opts.auth) { - headers['Authorization'] = opts.auth; + if ('authOption' in policy) { + const auth = opts[policy.authOption]; + if (typeof auth === 'string' && auth) { + headers.Authorization = auth; + } } - return { queryUrl: queryEndpoint, updateUrl: updateEndpoint, headers }; + return { queryUrl, updateUrl, headers }; } export interface LargeLiteralStorageConfig { @@ -256,22 +272,40 @@ export interface TripleStoreConfig { changelog?: boolean | ChangelogStoreOptions; } +export type TripleStoreFactoryConfig = TripleStoreConfig; + type AdapterFactory = ( options?: Record, ) => Promise; const adapterRegistry = new Map(); -export function registerTripleStoreAdapter( - name: string, +// Runtime compatibility guard for callers compiled before the worker adapter +// was removed. This is intentionally not part of STORAGE_ADAPTERS: it provides +// a migration error without making retired daemon/config policy constructible. +const REMOVED_ADAPTER_GUIDANCE: Readonly> = { + 'oxigraph-worker': + 'Use "sparql-http" or "blazegraph" for an HTTP store, or ' + + '"oxigraph-persistent" for embedded persistence.', +}; + +export function registerTripleStoreAdapter( + name: Backend, factory: AdapterFactory, -): void { +): Backend extends StorageAdapterBackend ? Backend : CustomTripleStoreBackend { adapterRegistry.set(name, factory); + return name as Backend extends StorageAdapterBackend ? Backend : CustomTripleStoreBackend; } export async function createTripleStore( - config: TripleStoreConfig, + config: TripleStoreFactoryConfig, ): Promise { + const removedGuidance = REMOVED_ADAPTER_GUIDANCE[config.backend]; + if (removedGuidance) { + throw new Error( + `TripleStore backend "${config.backend}" is no longer supported. ${removedGuidance}`, + ); + } const factory = adapterRegistry.get(config.backend); if (!factory) { throw new Error( @@ -337,8 +371,7 @@ function shouldEnableGraphSetIndex(config: TripleStoreConfig): boolean { function isDefaultLocalGraphSetIndexBackend(backend: TripleStoreBackend): boolean { return backend === 'oxigraph' - || backend === 'oxigraph-persistent' - || backend === 'oxigraph-worker'; + || backend === 'oxigraph-persistent'; } function resolveLargeLiteralStorageOptions( diff --git a/packages/storage/test/graph-set-index-store.test.ts b/packages/storage/test/graph-set-index-store.test.ts index 751b937fe2..67aaecb55e 100644 --- a/packages/storage/test/graph-set-index-store.test.ts +++ b/packages/storage/test/graph-set-index-store.test.ts @@ -727,8 +727,10 @@ describe('GraphSetIndexStore', () => { }); it('leaves custom backends uncached unless explicitly enabled', async () => { - const backend = 'custom-remote-graph-set-index-test'; - registerTripleStoreAdapter(backend, async () => new OxigraphStore()); + const backend = registerTripleStoreAdapter( + 'custom-remote-graph-set-index-test', + async () => new OxigraphStore(), + ); const defaultStore = await createTripleStore({ backend }); expect(defaultStore.listGraphsByPrefix).toBeUndefined(); diff --git a/packages/storage/test/is-external-backend.test.ts b/packages/storage/test/is-external-backend.test.ts index 759e08ec35..daae5378d4 100644 --- a/packages/storage/test/is-external-backend.test.ts +++ b/packages/storage/test/is-external-backend.test.ts @@ -18,7 +18,6 @@ describe('isExternalBackend', () => { it('returns false for the oxigraph family', () => { expect(isExternalBackend('oxigraph')).toBe(false); - expect(isExternalBackend('oxigraph-worker')).toBe(false); expect(isExternalBackend('oxigraph-persistent')).toBe(false); }); diff --git a/packages/storage/test/oxigraph-worker-resilience.test.ts b/packages/storage/test/oxigraph-worker-resilience.test.ts deleted file mode 100644 index 89c7c6cb98..0000000000 --- a/packages/storage/test/oxigraph-worker-resilience.test.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { OxigraphWorkerStore, createTripleStore, type Quad } from '../src/index.js'; - -// These exercise the embedded worker adapter's resilience guards added to stop -// a single slow/wedged store op from hanging every other store-backed request -// behind it (issues #997 / #999 / #1002 / #1005 / #1008). They need the -// compiled worker artifact (`dist/adapters/oxigraph-worker-impl.js`); if it's -// missing we fail loudly with the remediation hint rather than silently skip, -// matching `storage.test.ts`'s convention. -// -// Design note: the per-op timeout is applied to READ-ONLY ops only. A read is -// side-effect-free, so rejecting it after the bound is a clean, determinate -// failure that surfaces a wedged worker on the exact paths that hang in prod. -// Mutations are left unbounded — timing one out would only drop the caller's -// promise while the write is still in flight, which the rest of the codebase -// would mis-read as a clean failure. So the timeout tests provoke a timeout by -// queuing a READ behind a busy worker, not by bounding a write. -function makeStore(opts?: { operationTimeoutMs?: number }, persistPath?: string): OxigraphWorkerStore { - try { - return new OxigraphWorkerStore(persistPath, opts); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (/oxigraph-worker-impl/.test(msg)) { - throw new Error( - `oxigraph-worker adapter is not runnable — run ` + - `\`pnpm --filter @origintrail-official/dkg-storage build\` first. Underlying: ${msg}`, - ); - } - throw err; - } -} - -// For the timeout tests the worker is left mid-operation; close() forcibly -// terminates the thread, but the graceful close reply may itself time out, so -// swallow any error during teardown. -async function closeQuietly(store: OxigraphWorkerStore): Promise { - await store.close().catch(() => {}); -} - -function quads(n: number): Quad[] { - const out: Quad[] = new Array(n); - for (let i = 0; i < n; i += 1) { - out[i] = { - subject: `urn:test:s:${i}`, - predicate: 'http://schema.org/name', - object: `"v${i}"`, - graph: 'urn:test:g', - }; - } - return out; -} - -function abortDuringListenerRegistration(message: string): AbortSignal { - let aborted = false; - let reason: Error | undefined; - return { - get aborted() { - return aborted; - }, - get reason() { - return reason; - }, - addEventListener(type: string, listener: EventListenerOrEventListenerObject) { - if (type !== 'abort') return; - aborted = true; - reason = new Error(message); - if (typeof listener === 'function') listener(new Event('abort')); - else listener.handleEvent(new Event('abort')); - }, - removeEventListener: () => undefined, - dispatchEvent: () => true, - onabort: null, - } as unknown as AbortSignal; -} - -// Occupy the single worker thread with a large UNBOUNDED insert (mutations are -// not bounded by the timeout), so a read posted right after it queues behind a -// busy worker — the production "wedged worker" signature in miniature. -const BUSY_QUERY = 'ASK { GRAPH { ?s ?p ?o } }'; - -describe('OxigraphWorkerStore resilience', () => { - it('rejects a READ queued behind a busy worker after operationTimeoutMs', async () => { - // 50k inserts take hundreds of ms; a 5ms bound on the queued read must - // reject well before the worker frees up. - const store = makeStore({ operationTimeoutMs: 5 }); - try { - const busy = store.insert(quads(50_000)); // occupies the worker (unbounded) - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('surfaces the timeout as a typed OXIGRAPH_WORKER_OP_TIMEOUT error', async () => { - const store = makeStore({ operationTimeoutMs: 5 }); - try { - const busy = store.insert(quads(50_000)); - const err: any = await store.query(BUSY_QUERY).then(() => null, (e) => e); - expect(err).toBeTruthy(); - expect(err.code).toBe('OXIGRAPH_WORKER_OP_TIMEOUT'); - expect(err.method).toBe('query'); - expect(err.timeoutMs).toBe(5); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('rejects a queued READ promptly when the caller aborts', async () => { - const store = makeStore({ operationTimeoutMs: 60_000 }); - try { - const busy = store.insert(quads(50_000)); - const controller = new AbortController(); - const read = store.query(BUSY_QUERY, { signal: controller.signal }); - controller.abort(new Error('caller aborted queued read')); - await expect(read).rejects.toThrow(/caller aborted queued read/); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('rejects when the caller aborts while registering the abort listener', async () => { - const store = makeStore({ operationTimeoutMs: 60_000 }); - try { - await expect( - store.query(BUSY_QUERY, { signal: abortDuringListenerRegistration('listener registration aborted') }), - ).rejects.toThrow(/listener registration aborted/); - } finally { - await closeQuietly(store); - } - }); - - it('does NOT bound mutations — a large insert under a tiny timeout still completes', async () => { - // The whole point of read-only scoping: a write is never reported as a clean - // failure while it's still in flight. With a 5ms bound a 50k insert (>>5ms) - // would reject if it were bounded; it must resolve instead. - const store = makeStore({ operationTimeoutMs: 5 }); - try { - await expect(store.insert(quads(50_000))).resolves.toBeUndefined(); - } finally { - await closeQuietly(store); - } - }); - - it('completes normally within a generous timeout', async () => { - const store = makeStore({ operationTimeoutMs: 60_000 }); - try { - await store.insert(quads(10)); - expect(await store.countQuads('urn:test:g')).toBe(10); - // The data lives in a NAMED graph, so the ASK must scope to it (a bare - // `ASK { ?s ?p ?o }` only matches the default graph). - const r = await store.query(BUSY_QUERY); - expect(r.type).toBe('boolean'); - if (r.type === 'boolean') expect(r.value).toBe(true); - } finally { - await closeQuietly(store); - } - }); - - it('operationTimeoutMs: 0 disables the timeout (a queued read still completes)', async () => { - const store = makeStore({ operationTimeoutMs: 0 }); - try { - const busy = store.insert(quads(50_000)); - // With the bound disabled, the read WAITS for the worker instead of - // rejecting, then returns true once the import lands. - const r = await store.query(BUSY_QUERY); - expect(r.type).toBe('boolean'); - if (r.type === 'boolean') expect(r.value).toBe(true); - await busy; - expect(await store.countQuads('urn:test:g')).toBe(50_000); - } finally { - await closeQuietly(store); - } - }); - - it('a fractional operationTimeoutMs is floored to an integer', async () => { - // normalizeNonNegativeInt floors the ms bound: 5.9 behaves like 5, so the - // surfaced error reports "after 5ms" (no fractional noise). - const store = makeStore({ operationTimeoutMs: 5.9 }); - try { - const busy = store.insert(quads(50_000)); - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('close() is exempt from the per-op timeout so the final flush is never cut short', async () => { - // Codex review: close runs the worker's final flush; bounding it by the - // per-op timeout could terminate() the thread mid-flush and lose writes. - // With a tiny operationTimeoutMs and the worker still busy on a large - // insert, close() must WAIT for the worker to drain rather than reject — - // AND the in-flight write must actually land, not get killed mid-flush. - // We prove durability end-to-end on a persistent path: the regression this - // guards (close-after-debounced-flush race) was a SILENT data loss, so the - // test must assert the quads survive a reopen, not just that close resolves. - const dir = mkdtempSync(join(tmpdir(), 'oxigraph-worker-close-')); - const path = join(dir, 'store.nq'); - try { - const store = makeStore({ operationTimeoutMs: 10 }, path); - const busy = store.insert(quads(50_000)); // occupies the worker - // A read queued behind it times out at 10ms — proves the worker is busy. - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out/); - // close() must resolve cleanly (not reject with a 10ms timeout): it waits - // for the in-flight op to drain, then flushes + terminates. - await expect(store.close()).resolves.toBeUndefined(); - // The in-flight insert must have COMPLETED (drained), not been terminated - // mid-flight — otherwise close() silently cut it short. - await expect(busy).resolves.toBeUndefined(); - // And the write must be durable: a fresh worker on the same path hydrates - // all 50k quads, proving close() flushed before terminating. - const reopened = makeStore({ operationTimeoutMs: 60_000 }, path); - try { - expect(await reopened.countQuads('urn:test:g')).toBe(50_000); - } finally { - await closeQuietly(reopened); - } - } finally { - try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } - } - }); - - it('concurrent and repeated close() calls all resolve without hanging', async () => { - // Codex review: close() goes through an UNBOUNDED worker RPC, so a second - // close racing the first was orphaned when terminate() killed the worker and - // never settled. close() is now memoized + the worker 'exit' handler rejects - // anything still pending, so concurrent/repeat closes all settle. - const store = makeStore({ operationTimeoutMs: 60_000 }); - await store.insert(quads(5)); - const results = await Promise.all([store.close(), store.close(), store.close()]); - expect(results).toEqual([undefined, undefined, undefined]); - // Ops issued after close fail fast (store closed) instead of hanging. - await new Promise((r) => setImmediate(r)); - await expect(store.insert(quads(1))).rejects.toThrow(/closed/i); - }); - - it('store.options reach the adapter through createTripleStore (factory path)', async () => { - // Codex review: the user-facing path is createTripleStore({ backend, options }), - // not the constructor — assert the option forwarding in the adapter factory - // actually takes effect so a typo there can't silently drop the knob. - // operationTimeoutMs forwarded: a 5ms bound rejects a read queued behind a - // busy worker. - const store = await createTripleStore({ - backend: 'oxigraph-worker', - options: { operationTimeoutMs: 5 }, - }); - try { - const busy = store.insert(quads(50_000)); - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); - await busy.catch(() => {}); - } finally { - await store.close().catch(() => {}); - } - }); -}); diff --git a/packages/storage/test/oxigraph-worker-respawn.test.ts b/packages/storage/test/oxigraph-worker-respawn.test.ts deleted file mode 100644 index 6d60206177..0000000000 --- a/packages/storage/test/oxigraph-worker-respawn.test.ts +++ /dev/null @@ -1,348 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import type { Worker } from 'node:worker_threads'; -import { OxigraphWorkerStore, type Quad } from '../src/index.js'; - -// Regression tests for unexpected-worker-exit recovery. The production -// failure: ERR_WORKER_OUT_OF_MEMORY kills ONLY the worker thread, the daemon -// process keeps running, and the old adapter latched `workerExited` forever — -// a testnet node served HTTP for DAYS with a dead store (green /api/status, -// 503 on every write). The fix auto-respawns the worker on an exit that was -// NOT initiated by close(); these tests kill the REAL worker thread (no -// mocks) by reaching into the private `worker` field and calling terminate(), -// which raises the exact same 'exit'-without-close signal as an OOM kill. -// -// Like the resilience suite next door, this needs the compiled worker -// artifact (`dist/adapters/oxigraph-worker-impl.js`); if it's missing we fail -// loudly with the remediation hint rather than silently skip. -function makeStore(persistPath?: string, opts?: { operationTimeoutMs?: number }): OxigraphWorkerStore { - try { - return new OxigraphWorkerStore(persistPath, opts); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (/oxigraph-worker-impl/.test(msg)) { - throw new Error( - `oxigraph-worker adapter is not runnable — run ` + - `\`pnpm --filter @origintrail-official/dkg-storage build\` first. Underlying: ${msg}`, - ); - } - throw err; - } -} - -// The recovery machinery is intentionally private (nothing outside the -// adapter should steer it), so the tests reach in through one typed seam -// instead of scattering `as any` casts around. `lifecycle` is the explicit -// state field the respawn/close/in-memory-loss logic keys off (see the -// WorkerLifecycle model in the adapter); asserting on it pins the state model. -type WorkerLifecycle = - | 'initializing' - | 'live' - | 'respawning' - | 'closing' - | 'closed' - | 'gave_up' - | 'in_memory_lost'; -function internals(store: OxigraphWorkerStore): { - worker: Worker; - lifecycle: WorkerLifecycle; - closePromise: Promise | null; - respawnPromise: Promise | null; - consecutiveRespawnFailures: number; -} { - return store as unknown as { - worker: Worker; - lifecycle: WorkerLifecycle; - closePromise: Promise | null; - respawnPromise: Promise | null; - consecutiveRespawnFailures: number; - }; -} - -/** - * Simulate an OOM-style death of the CURRENT worker thread: terminate() while - * closePromise is null fires the same 'exit' event a crashed thread does. - * Awaiting terminate() guarantees the store's own 'exit' handler already ran - * (it was registered first, and 'exit' listeners run synchronously at emit), - * so on return the respawn has been scheduled — or already completed, for the - * immediate first attempt. - */ -async function killWorker(store: OxigraphWorkerStore): Promise { - await internals(store).worker.terminate(); -} - -function quads(n: number, graph = 'urn:test:g'): Quad[] { - const out: Quad[] = new Array(n); - for (let i = 0; i < n; i += 1) { - out[i] = { - subject: `urn:test:s:${i}`, - predicate: 'http://schema.org/name', - object: `"v${i}"`, - graph, - }; - } - return out; -} - -describe('OxigraphWorkerStore respawn after unexpected worker exit', () => { - let dir: string; - let path: string; - - beforeEach(() => { - dir = mkdtempSync(join(tmpdir(), 'oxigraph-worker-respawn-')); - path = join(dir, 'store.nq'); - }); - - afterEach(() => { - try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } - }); - - it('recovers from an unexpected worker death — subsequent ops succeed on a respawned worker', async () => { - const store = makeStore(path); - try { - await store.insert(quads(10)); - // insert() only schedules the worker's 50ms debounced flush; force the - // data to disk so the respawned worker hydrates it back. - await store.flush(); - - const before = internals(store).worker; - await killWorker(store); - - // Same store OBJECT (every daemon alias keeps working), new thread. - expect(internals(store).worker).not.toBe(before); - - // Reads see the persisted data again — the exact op that used to fail - // forever with "the store is closed". - expect(await store.countQuads('urn:test:g')).toBe(10); - // A disk-persisted store recovers fully: the state model is back to 'live' - // (this is the branch finding 1 deliberately keeps auto-respawning). - expect(internals(store).lifecycle).toBe('live'); - // And writes work too: the store is fully live, not read-only. - await store.insert(quads(5, 'urn:test:g2')); - expect(await store.countQuads('urn:test:g2')).toBe(5); - } finally { - await store.close().catch(() => {}); - } - }); - - it('rejects an in-flight op at kill time with the retryable "restarted — retry" message', async () => { - const store = makeStore(path); - try { - // A 50k insert occupies the single worker thread for hundreds of ms, so - // terminating right after posting is guaranteed to catch it in flight. - const inflight = store.insert(quads(50_000)); - await killWorker(store); - // The outcome of the killed op is genuinely unknown (the thread died - // mid-processing), so the caller gets a RETRYABLE error, not "closed". - await expect(inflight).rejects.toThrow(/oxigraph-worker restarted — retry/); - // ...and a retry against the same store object then succeeds. - await expect(store.insert(quads(10))).resolves.toBeUndefined(); - expect(await store.countQuads('urn:test:g')).toBe(10); - } finally { - await store.close().catch(() => {}); - } - }); - - it('parks ops issued during a respawn backoff instead of failing them', async () => { - const store = makeStore(path); - try { - await store.insert(quads(10)); - await store.flush(); - - // First kill → immediate respawn (backoff tier 0). Kill the replacement - // BEFORE it serves any op, so the second respawn sits in the 1s backoff - // tier — that's the window where new ops must park, not fail. - await killWorker(store); - await killWorker(store); - expect(internals(store).respawnPromise).not.toBeNull(); - - // Issued mid-backoff: the op waits for the replacement worker and then - // succeeds against the disk-persisted state (slightly slower success, - // never a spurious "store is closed"). - expect(await store.countQuads('urn:test:g')).toBe(10); - expect(internals(store).respawnPromise).toBeNull(); - } finally { - await store.close().catch(() => {}); - } - }, 15_000); - - it('a successful op resets the crash-loop counter', async () => { - const store = makeStore(path); - try { - await store.insert(quads(3)); - await store.flush(); - await killWorker(store); - // The respawn consumed one attempt... - expect(internals(store).consecutiveRespawnFailures).toBe(1); - // ...and the first successful reply proves the worker healthy, ending - // the accounting window (occasional crashes days apart never accumulate - // toward the give-up bound). - expect(await store.countQuads('urn:test:g')).toBe(3); - expect(internals(store).consecutiveRespawnFailures).toBe(0); - } finally { - await store.close().catch(() => {}); - } - }); - - it('close() does NOT respawn — the exit stays intentional and ops fail closed', async () => { - const store = makeStore(path); - await store.insert(quads(5)); - const lastWorker = internals(store).worker; - await store.close(); - - // Give any (buggy) respawn scheduling a chance to run before asserting. - await new Promise((r) => setTimeout(r, 50)); - expect(internals(store).respawnPromise).toBeNull(); - expect(internals(store).worker).toBe(lastWorker); - - // Post-close ops fail exactly as before this fix — fast and permanent. - await expect(store.countQuads('urn:test:g')).rejects.toThrow(/store is closed/); - await expect(store.insert(quads(1))).rejects.toThrow(/store is closed/); - }); - - it('gives up after MAX consecutive dead-on-arrival respawns and latches closed with guidance', async () => { - const store = makeStore(path); - try { - await store.insert(quads(2)); - await store.flush(); - // Simulate a genuine crash loop without waiting out the real 1s/5s/30s - // backoff ladder: pre-load the consecutive-failure counter to the bound, - // so the very next unexpected exit hits the give-up branch immediately. - internals(store).consecutiveRespawnFailures = 5; - await killWorker(store); - - // Latched closed — same fail-fast as the pre-recovery behaviour, but the - // error now tells the operator WHY and what to do about it. - const err: any = await store.countQuads('urn:test:g').then(() => null, (e) => e); - expect(err).toBeTruthy(); - expect(err.message).toMatch(/store is closed/); - expect(err.message).toMatch(/respawn gave up/); - // No further respawns get armed by later traffic. - expect(internals(store).respawnPromise).toBeNull(); - // The give-up verdict is now the single, explicit terminal state — no - // cluster of booleans that could disagree with the fail-fast error above. - expect(internals(store).lifecycle).toBe('gave_up'); - } finally { - // close() on a latched store must still resolve (idempotent teardown). - await expect(store.close()).resolves.toBeUndefined(); - } - }); - - it('close() during a respawn backoff wins — the store stays closed', async () => { - const store = makeStore(path); - await store.insert(quads(4)); - await store.flush(); - // Double-kill parks the second respawn in the 1s backoff tier (as above). - await killWorker(store); - await killWorker(store); - expect(internals(store).respawnPromise).not.toBeNull(); - - // An operator close arriving mid-backoff must never be resurrected by the - // supervisor: the pending respawn bails out when it wakes. - await expect(store.close()).resolves.toBeUndefined(); - // Wait out the backoff so a buggy respawn would have fired by now. - await new Promise((r) => setTimeout(r, 1_500)); - await expect(store.countQuads('urn:test:g')).rejects.toThrow(/store is closed/); - }, 15_000); -}); - -// Finding 1 (🔴): an IN-MEMORY store (constructed with NO persistPath) has no -// disk to reload from, so respawning after an unexpected worker exit would come -// up EMPTY yet look perfectly healthy — every row written before the crash -// silently gone (confirmed live: pre-crash data vanished). The fix is to FAIL -// CLOSED on such a store instead of respawning: the worker's data is -// unrecoverable, so subsequent ops must reject with a clear data-loss error, -// never return a wrong-but-plausible empty result. These tests use the REAL -// worker thread (no mocks) exactly like the persistent-store suite above, -// killing it via terminate() to raise the same 'exit'-without-close signal an -// OOM kill would. -describe('OxigraphWorkerStore in-memory fail-closed on unexpected worker exit', () => { - it('fails closed with a data-loss error after an in-memory worker crash — never a silent empty result', async () => { - // No persistPath → in-memory store. This is the exact shape that silently - // lost data before the fix. - const store = makeStore(undefined); - try { - await store.insert(quads(7)); - // Sanity: the data really is there before the crash. - expect(await store.countQuads('urn:test:g')).toBe(7); - - const before = internals(store).worker; - await killWorker(store); - - // The store must NOT respawn (no disk to recover from) — it latches into - // the terminal in_memory_lost state and never arms a replacement worker. - expect(internals(store).lifecycle).toBe('in_memory_lost'); - expect(internals(store).respawnPromise).toBeNull(); - expect(internals(store).worker).toBe(before); // no fresh (empty) thread - - // The regression this guards: a read here used to RESOLVE with 0 (an - // empty respawned store), reporting SUCCESS while all 7 rows were gone. - // `.then(() => null, e => e)` captures a resolve as null and a reject as - // the error, so a truthy `err` proves the read rejected rather than - // silently returning the wrong count. - const err: any = await store.countQuads('urn:test:g').then(() => null, (e) => e); - expect(err).toBeTruthy(); - expect(err.message).toMatch(/IN-MEMORY store's worker crashed/); - expect(err.message).toMatch(/data was lost|cannot be recovered/); - - // Writes fail closed the same way — the store is unusable, not read-only. - await expect(store.insert(quads(1))).rejects.toThrow(/IN-MEMORY store's worker crashed/); - } finally { - // close() on a data-lost store must still resolve (idempotent teardown) - // and must not resurrect it. - await expect(store.close()).resolves.toBeUndefined(); - } - }); - - it('the in_memory_lost state is terminal — later traffic never arms a respawn', async () => { - const store = makeStore(undefined); - try { - await store.insert(quads(3)); - await killWorker(store); - expect(internals(store).lifecycle).toBe('in_memory_lost'); - - // Hammer it with a few more ops: each must fail fast, and none may flip - // the store back to 'respawning'/'live' (the terminal-state guarantee). - for (let i = 0; i < 3; i += 1) { - await expect(store.countQuads('urn:test:g')).rejects.toThrow(/cannot be recovered/); - expect(internals(store).lifecycle).toBe('in_memory_lost'); - expect(internals(store).respawnPromise).toBeNull(); - } - } finally { - await store.close().catch(() => {}); - } - }); - - it('otReviewAgent #1408: the spawn→live transition is guarded — a spawn from a terminal state THROWS (never resurrects the store), but the two legal predecessors enter live', async () => { - // The spawn path (spawnWorker → markSpawnedLive) is the one writer that - // enters 'live'. It must still enforce terminal permanence: were a future - // respawn/close code path to call it after close/give-up/in-memory-loss, it - // has to throw rather than silently reopen the store. Poke the state field - // directly (decoupled from the real worker) to drive the guard. - const store = makeStore(undefined); - const seam = store as unknown as { - lifecycle: WorkerLifecycle; - markSpawnedLive: () => void; - }; - try { - for (const terminal of ['closed', 'gave_up', 'in_memory_lost'] as const) { - seam.lifecycle = terminal; - expect(() => seam.markSpawnedLive()).toThrow(/illegal spawn transition|terminal/i); - expect(seam.lifecycle).toBe(terminal); // stayed terminal — not resurrected - } - // 'closing' is likewise not a legal spawn predecessor. - seam.lifecycle = 'closing'; - expect(() => seam.markSpawnedLive()).toThrow(/illegal spawn transition/i); - // The two legal predecessors DO enter 'live'. - for (const start of ['initializing', 'respawning'] as const) { - seam.lifecycle = start; - expect(() => seam.markSpawnedLive()).not.toThrow(); - expect(seam.lifecycle).toBe('live'); - } - } finally { - await store.close().catch(() => {}); - } - }); -}); diff --git a/packages/storage/test/storage.test.ts b/packages/storage/test/storage.test.ts index d9188f06f2..8ac362dca2 100644 --- a/packages/storage/test/storage.test.ts +++ b/packages/storage/test/storage.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, expectTypeOf, beforeEach, beforeAll, afterAll } from 'vitest'; import { OxigraphStore, BlazegraphStore, @@ -6,6 +6,8 @@ import { GraphManager, PrivateContentStore, createTripleStore, + classifyTripleStoreBackend, + customTripleStoreBackend, loadSelectedSharedMemoryQuads, loadSelectedVerifiableMemoryQuads, registerTripleStoreAdapter, @@ -13,6 +15,7 @@ import { resolveVerifiableMemoryReadGraphs, type Quad, type TripleStore, + type TripleStoreBackend, } from '../src/index.js'; import { contextGraphDataGraphUri, @@ -252,12 +255,17 @@ if (blazeUrl) { // --------------------------------------------------------------------------- describe('createTripleStore factory', () => { + it('keeps managed and retired daemon names out of the constructible backend type', () => { + expectTypeOf<'oxigraph'>().toMatchTypeOf(); + expectTypeOf<'oxigraph-server'>().not.toMatchTypeOf(); + expectTypeOf<'oxigraph-worker'>().not.toMatchTypeOf(); + }); + it('all built-in backends are registered (factory throws something other than "Unknown TripleStore backend")', async () => { // The *registry* contract being tested here is: every built-in // backend name is recognized. The construction itself may require - // options (blazegraph needs `url`, sparql-http needs `queryEndpoint`) - // or worker artifacts (oxigraph-worker needs the compiled worker - // impl). So a backend passes this test iff calling `createTripleStore` + // options (blazegraph needs `url`, sparql-http needs `queryEndpoint`). + // So a backend passes this test iff calling `createTripleStore` // either succeeds OR throws a *non*-"Unknown TripleStore backend" // error. // @@ -266,7 +274,7 @@ describe('createTripleStore factory', () => { // effectively assert "a promise settled" — noise. This version // asserts the positive contract explicitly and points at the // specific failing backend if the registry regresses. - const backends = ['oxigraph', 'oxigraph-worker', 'blazegraph', 'sparql-http']; + const backends = ['oxigraph', 'blazegraph', 'sparql-http']; for (const backend of backends) { let outcome: 'constructed' | Error; try { @@ -312,56 +320,43 @@ describe('createTripleStore factory', () => { ).rejects.toThrow('queryEndpoint'); }); - it('oxigraph-worker adapter is registered and round-trips an insert', async () => { - // The worker adapter resolves `./oxigraph-worker-impl.js` relative to - // the module loaded at runtime. When vitest runs against raw source - // without a prior `pnpm build`, that URL lands in `src/adapters/` where - // only the .ts files live, so the Worker constructor throws - // "Cannot find module … oxigraph-worker-impl.js". - // - // This used to be caught and converted to `ctx.skip()`, which meant a - // green CI run even when the worker artifact was missing — i.e. a - // broken build never triggered a test failure. We now FAIL LOUDLY in - // that case with a remediation hint, so: - // • locally, the developer sees "run pnpm build first" instead of a - // silent skip; - // • in CI, if `pnpm build` was not wired into the lane (or the build - // regresses), this test surfaces it as a red failure. - let store: Awaited>; - try { - store = await createTripleStore({ backend: 'oxigraph-worker' }); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - if (msg.includes('Cannot find module') && msg.includes('oxigraph-worker-impl')) { - throw new Error( - `oxigraph-worker adapter is not runnable — the compiled ` + - `oxigraph-worker-impl.js artifact is missing from ` + - `packages/storage/dist/adapters/. Run ` + - `\`pnpm --filter @origintrail-official/dkg-storage build\` ` + - `before running this test. Underlying error: ${msg}`, - ); - } - throw err; - } - await store.insert([{ - subject: 'http://ex.org/s', - predicate: 'http://ex.org/p', - object: '"hi"', - graph: 'http://ex.org/g', - }]); - expect(await store.countQuads()).toBe(1); - await store.close(); - }); - it('throws on unknown backend', async () => { - await expect(createTripleStore({ backend: 'unknown' })).rejects.toThrow( + await expect(createTripleStore({ backend: customTripleStoreBackend('unknown') })).rejects.toThrow( 'Unknown TripleStore backend', ); }); + it('gives runtime migration guidance to legacy oxigraph-worker callers', async () => { + await expect(createTripleStore({ + backend: 'oxigraph-worker' as any, + })).rejects.toThrow( + /oxigraph-worker.*no longer supported.*sparql-http.*oxigraph-persistent/, + ); + }); + + it('classifies only factory adapters and leaves daemon policy names outside storage', async () => { + expect(classifyTripleStoreBackend('oxigraph')).toEqual({ + kind: 'adapter', + backend: 'oxigraph', + }); + expect(classifyTripleStoreBackend('oxigraph-server')).toEqual({ + kind: 'custom', + backend: 'oxigraph-server', + }); + expect(classifyTripleStoreBackend('oxigraph-worker')).toEqual({ + kind: 'custom', + backend: 'oxigraph-worker', + }); + await expect(createTripleStore({ + backend: customTripleStoreBackend('oxigraph-server'), + })).rejects.toThrow( + /Unknown TripleStore backend/, + ); + }); + it('custom adapter can be registered and used', async () => { const calls: string[] = []; - registerTripleStoreAdapter('test-custom', async () => ({ + const backend = registerTripleStoreAdapter('test-custom', async () => ({ insert: async () => { calls.push('insert'); }, delete: async () => {}, deleteByPattern: async () => 0, @@ -375,7 +370,7 @@ describe('createTripleStore factory', () => { close: async () => {}, })); - const store = await createTripleStore({ backend: 'test-custom' }); + const store = await createTripleStore({ backend }); await store.insert([]); expect(calls).toEqual(['insert']); expect(await store.countQuads()).toBe(1); diff --git a/scripts/devnet.sh b/scripts/devnet.sh index ff21784c35..03fabea9a3 100755 --- a/scripts/devnet.sh +++ b/scripts/devnet.sh @@ -546,7 +546,7 @@ create_node_config() { # and spawns it on its own port — NO Docker. Node 1 is the node the # UI/e2e suite drives, so the suite now exercises the REAL default # backend and deterministically reproduces SPARQL-over-HTTP-only - # bugs such as #996 — which the old `oxigraph-worker` default hid.) + # bugs such as #996.) # Node 3-4: blazegraph (if Docker) else oxigraph (in-process baseline) # Node 5-6: sparql-http → external Dockerized Oxigraph (EXTRA coverage of the # generic external-endpoint path; Docker-only, optional) @@ -573,6 +573,8 @@ create_node_config() { local ox_port_var="OXIGRAPH_SERVER_PORT_${node_num}" local ox_port="${!ox_port_var}" store_block="\"store\": { \"backend\": \"sparql-http\", \"options\": { \"queryEndpoint\": \"http://127.0.0.1:${ox_port}/query\", \"updateEndpoint\": \"http://127.0.0.1:${ox_port}/update\" } }," + else + store_block="\"store\": { \"backend\": \"oxigraph\" }," fi fi @@ -1769,12 +1771,12 @@ cmd_start() { local api_port=$((API_PORT_BASE + i - 1)) local role="edge" [ "$i" -le "$NUM_CORE_NODES" ] && role="core" - local store_label="oxigraph-worker" + local store_label="oxigraph-server" if [ "$i" -ge 3 ] && [ "$i" -le 4 ]; then [ "$BLAZEGRAPH_AVAILABLE" = true ] && store_label="blazegraph" || store_label="oxigraph" fi if [ "$i" -ge 5 ]; then - [ "$OXIGRAPH_SERVER_AVAILABLE" = true ] && store_label="oxigraph-server" || store_label="oxigraph-worker" + [ "$OXIGRAPH_SERVER_AVAILABLE" = true ] && store_label="sparql-http" || store_label="oxigraph" fi log "Node $i ($role, $store_label): http://127.0.0.1:$api_port/ui" done diff --git a/scripts/publisher-smoke-test.sh b/scripts/publisher-smoke-test.sh index 1086fe0cc5..28472b5660 100755 --- a/scripts/publisher-smoke-test.sh +++ b/scripts/publisher-smoke-test.sh @@ -76,7 +76,7 @@ import { DKGPublisher, TripleStoreAsyncLiftPublisher } from '@origintrail-offici const dkgHome = process.env.DKG_HOME; const privateKey = process.env.SMOKE_PRIVATE_KEY; const store = await createTripleStore({ - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(dkgHome, 'store.nq') }, }); From 676a2b8addb90bcdd75bd273ff93c91617aa53fa Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:10:26 +0200 Subject: [PATCH 10/12] Revert "Retire oxigraph-worker backend (#1539)" (#1637) This reverts commit 62ab6231d2a03b1b6beb62117fb8ecaa0e70dc0f. Co-authored-by: Branimir Rakic --- CHANGELOG.md | 4 - README.md | 8 +- bench/store-read-latency.bench.ts | 188 ++++- docs/use-dkg/storage-sparql-http.md | 4 +- packages/agent/src/dkg-agent-helpers.ts | 1 + packages/agent/src/dkg-agent-types.ts | 2 +- packages/agent/src/dkg-agent.ts | 4 +- .../agent/src/sync/responder/sync-handler.ts | 2 +- .../test/context-graph-discovery.test.ts | 6 +- packages/cli/src/commands/hermes.ts | 3 +- packages/cli/src/commands/init.ts | 10 +- packages/cli/src/commands/lifecycle.ts | 2 +- packages/cli/src/commands/mcp.ts | 3 +- packages/cli/src/commands/openclaw.ts | 3 +- packages/cli/src/config.ts | 57 +- packages/cli/src/daemon/chain-reset-wipe.ts | 129 ++- packages/cli/src/daemon/daemon-state.ts | 76 -- packages/cli/src/daemon/handle-request.ts | 12 +- packages/cli/src/daemon/lifecycle.ts | 60 +- packages/cli/src/daemon/oxigraph-managed.ts | 5 +- packages/cli/src/daemon/routes/context.ts | 26 +- packages/cli/src/daemon/routes/status.ts | 46 +- packages/cli/src/daemon/store-runtime.ts | 174 ----- packages/cli/src/publisher-runner.ts | 16 +- packages/cli/src/store-backends.ts | 226 ------ packages/cli/src/store-wizard.ts | 233 +++--- packages/cli/test/chain-reset-wipe.test.ts | 30 +- .../test/daemon-http-behavior-extra.test.ts | 2 +- .../cli/test/daemon-http-inflight-cap.test.ts | 171 ++-- .../test/daemon-startup-validation.test.ts | 190 +---- packages/cli/test/daemon-state.test.ts | 53 -- .../daemon-storage-ack-timing-wiring.test.ts | 2 - .../daemon-sync-agents-meta-wiring.test.ts | 1 - .../test/daemon/plugin-routes-api.e2e.test.ts | 2 +- .../handle-request-store-persistence.test.ts | 103 --- packages/cli/test/helpers/live-daemon.ts | 13 +- packages/cli/test/oxigraph-managed.test.ts | 4 +- .../cli/test/publisher-managed-store.test.ts | 22 - packages/cli/test/publisher-wallets.test.ts | 42 - packages/cli/test/status-route-rpc.test.ts | 173 +--- .../cli/test/store-backend-taxonomy.test.ts | 122 --- packages/cli/test/store-health-check.test.ts | 2 +- packages/cli/test/store-identity-tag.test.ts | 2 +- packages/cli/test/store-runtime.test.ts | 164 ---- packages/cli/test/store-wizard.test.ts | 120 +-- .../cli/test/validate-store-config.test.ts | 24 +- .../test/write-preflight-resilience.test.ts | 41 +- packages/cli/vitest.unit.config.ts | 4 +- packages/core/src/proto/storage-ack.ts | 6 +- .../core/test/ensure-dkg-node-config.test.ts | 2 +- .../test/kafka-plugin-api.e2e.test.ts | 2 +- .../devnet/publish-ack-quorum.devnet.spec.ts | 2 +- .../write-preflight-guard.devnet.spec.ts | 4 +- packages/query/src/query-handler.ts | 2 +- packages/storage/README.md | 34 + .../src/adapters/oxigraph-worker-impl.ts | 18 + .../storage/src/adapters/oxigraph-worker.ts | 737 ++++++++++++++++++ packages/storage/src/adapters/oxigraph.ts | 4 +- packages/storage/src/index.ts | 23 +- packages/storage/src/store-backends.ts | 84 -- packages/storage/src/triple-store.ts | 101 +-- .../test/graph-set-index-store.test.ts | 6 +- .../storage/test/is-external-backend.test.ts | 1 + .../test/oxigraph-worker-resilience.test.ts | 261 +++++++ .../test/oxigraph-worker-respawn.test.ts | 348 +++++++++ packages/storage/test/storage.test.ts | 93 +-- scripts/devnet.sh | 8 +- scripts/publisher-smoke-test.sh | 2 +- 68 files changed, 2161 insertions(+), 2164 deletions(-) delete mode 100644 packages/cli/src/daemon/daemon-state.ts delete mode 100644 packages/cli/src/daemon/store-runtime.ts delete mode 100644 packages/cli/src/store-backends.ts delete mode 100644 packages/cli/test/daemon-state.test.ts delete mode 100644 packages/cli/test/handle-request-store-persistence.test.ts delete mode 100644 packages/cli/test/publisher-managed-store.test.ts delete mode 100644 packages/cli/test/store-backend-taxonomy.test.ts delete mode 100644 packages/cli/test/store-runtime.test.ts create mode 100644 packages/storage/src/adapters/oxigraph-worker-impl.ts create mode 100644 packages/storage/src/adapters/oxigraph-worker.ts delete mode 100644 packages/storage/src/store-backends.ts create mode 100644 packages/storage/test/oxigraph-worker-resilience.test.ts create mode 100644 packages/storage/test/oxigraph-worker-respawn.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d283a829ce..3ceea71cf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,6 @@ All notable changes to the DKG V10 node are documented here. The format is based ## [Unreleased] -### Removed — `oxigraph-worker` backend support - -- **The embedded `oxigraph-worker` backend is retired.** The storage package no longer exports or registers the worker-thread adapter, `createTripleStore({ backend: "oxigraph-worker" })` fails with an actionable migration error, and CLI config validation refuses explicit `store.backend: "oxigraph-worker"` before daemon boot. Block-less configs now resolve to the daemon-managed `oxigraph-server` default; if a legacy `store.nq` file exists, daemon boot requires `DKG_ACCEPT_STORE_RESET=1` so operators acknowledge the fresh-store cutover. - ## [10.0.6] - 2026-07-10 Sync-, storage-, and admission-path hardening on top of 10.0.5, plus the StorageACK priority-lane follow-ups and a set of CLI/RPC fixes. Eliminates the perpetually-dirty graph-set-index full scan that was saturating managed Oxigraph cores, bounds sync-responder memory and coalesces duplicate sync fan-out, makes network admission probing back off and use canonical peer ids, and completes the StorageACK priority-lane hardening (ACK candidate selection, async promote-queue read serialization, and OT-RFC-49 host-mode ciphertext strip-by-curation). **No smart-contract changes — no deployment required** (no Solidity source, ABI, or mainnet/testnet deployment-registry changes since 10.0.5). diff --git a/README.md b/README.md index 8fb7d724a0..cb9a5aa15e 100644 --- a/README.md +++ b/README.md @@ -403,13 +403,11 @@ analysis reports are under `bench/results/profiles/`, including ## Triple Store Backends -A DKG node keeps every assertion in an [RDF](https://www.w3.org/RDF/) triple store. Out of the box the daemon manages a local [Oxigraph](https://github.com/oxigraph/oxigraph) server, so a workstation needs no separate setup. Heavier deployments can swap in [Blazegraph](https://blazegraph.com/) (the mainnet store) or any SPARQL 1.1 server. +A DKG node keeps every assertion in an [RDF](https://www.w3.org/RDF/) triple store. Out of the box the node runs an embedded [Oxigraph](https://github.com/oxigraph/oxigraph) instance, which is everything you need on a workstation — no extra process, no extra port, no extra config. Heavier deployments can swap in [Blazegraph](https://blazegraph.com/) (the mainnet store) or any SPARQL 1.1 server. | Backend | When to pick it | |---|---| -| `oxigraph-server` (default) | Single-operator nodes, dev, CI. Managed automatically by the daemon with persistent local storage. | -| `oxigraph` | Embedded in-memory Oxigraph for development and short-lived tests. | -| `oxigraph-persistent` | Embedded persistent Oxigraph when an explicit existing store path is required. | +| `oxigraph-worker` (default) | Single-operator nodes, dev, CI. No setup. File-backed, capped at process RAM. | | `blazegraph` | High-throughput nodes, mainnet parity, very large graphs (10M+ quads). Run as a separate daemon (Docker or `java -jar`). Shares cleanly with V6 / V8 instances — DKG scopes its writes to the `did:dkg:context-graph:` named-graph prefix. | | `sparql-http` | Any SPARQL 1.1 Protocol server (Fuseki, GraphDB, Stardog, Neptune…). Bring your own URL + (optional) auth header. | @@ -422,7 +420,7 @@ Two paths: ``` $ dkg init … -Triple store backend (oxigraph-server / oxigraph / blazegraph) (oxigraph-server): blazegraph +Triple store backend (oxigraph / blazegraph) (oxigraph): blazegraph Blazegraph SPARQL endpoint URL: http://127.0.0.1:9999/bigdata/namespace/mynode/sparql Store endpoint reachable: blazegraph http://127.0.0.1:9999/bigdata/namespace/mynode/sparql ``` diff --git a/bench/store-read-latency.bench.ts b/bench/store-read-latency.bench.ts index 9f2ec700ce..28d16edd52 100644 --- a/bench/store-read-latency.bench.ts +++ b/bench/store-read-latency.bench.ts @@ -1,3 +1,5 @@ +import { existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { defineSuite } from 'esbench'; // Import the store classes + types from their SPECIFIC source modules rather // than the storage barrel: the barrel also re-exports GraphManager / @@ -6,6 +8,7 @@ import { defineSuite } from 'esbench'; // clean checkout. These adapter modules depend only on `oxigraph` + the local // triple-store types, so the bench needs nothing beyond the storage build. import { OxigraphStore } from '../packages/storage/src/adapters/oxigraph.ts'; +import { OxigraphWorkerStore } from '../packages/storage/src/adapters/oxigraph-worker.ts'; import type { Quad, QueryResult } from '../packages/storage/src/triple-store.ts'; import { GET_TOTAL_TRIPLES_SPARQL, parseRdfInt } from '../packages/cli/src/daemon/metrics-queries.ts'; import { benchAsyncWithHooks } from './support/esbench-case-hooks.ts'; @@ -21,11 +24,33 @@ import { benchAsyncWithHooks } from './support/esbench-case-hooks.ts'; * - the production `getTotalTriples` `COUNT(*)` aggregate the 30s metrics * collector runs (`packages/cli/src/daemon/lifecycle.ts`). * + * Backends (env `DKG_BENCH_STORE_BACKENDS`, comma-separated): + * - `inprocess` — `OxigraphStore`, no build step required. + * - `worker` — `OxigraphWorkerStore`, the PRODUCTION backend; requires + * `pnpm --filter @origintrail-official/dkg-storage build` first (it spawns a + * compiled worker artefact). + * Default: `inprocess`, plus `worker` automatically when its compiled artefact + * is present (so a built tree / CI measures both; an unbuilt tree degrades to + * `inprocess`-only instead of erroring). + * + * CONTENTION — the `(under write load)` cases — is measured ONLY on the `worker` + * backend, by design. The production read-starvation is a property of the + * single-writer oxigraph WORKER, whose message queue serialises a read behind + * queued writes — exactly what an out-of-process MVCC server (#938) relieves. + * The in-process `OxigraphStore` runs its insert/query work SYNCHRONOUSLY on one + * thread, so a read and a write can never truly overlap; a same-thread "writer" + * would only measure event-loop interleaving (not store contention) and could + * report misleadingly idle reads. So `inprocess` runs the idle read-latency + * baselines only. + * * Store sizes via env `DKG_BENCH_STORE_SIZES` (default `1k,50k`). */ +type Backend = 'inprocess' | 'worker'; + interface ReadStore { insert(quads: Quad[]): Promise; + delete(quads: Quad[]): Promise; query(sparql: string): Promise; close(): Promise; } @@ -38,6 +63,13 @@ const READ_LIMIT1 = `SELECT ?s WHERE { GRAPH <${GRAPH}> { ?s ?p ?o } } LIMIT 1`; // synthetic data lives in a named graph, so the `GRAPH ?g` branch carries the scan. const READ_TOTAL_TRIPLES = GET_TOTAL_TRIPLES_SPARQL; +// Bounded write churn: the background writer repeatedly inserts then deletes a +// fixed batch in a region disjoint from the pre-populated base, so it generates +// sustained write work WITHOUT drifting the store size (which would otherwise +// confound the getTotalTriples-under-load measurement). +const CHURN_BATCH = 50; +const CHURN_OFFSET = 1_000_000_000; + const STORE_SIZES: Record = { '1k': 1_000, '10k': 10_000, '50k': 50_000, '200k': 200_000 }; const INSERT_CHUNK = 1_000; @@ -79,6 +111,46 @@ function makeQuads(count: number, offset: number): Quad[] { return quads; } +// The compiled worker artefact sits next to the storage dist build; its presence +// means the `worker` backend can be constructed without throwing. +function workerArtifactAvailable(): boolean { + try { + return existsSync(fileURLToPath(new URL('../packages/storage/dist/adapters/oxigraph-worker-impl.js', import.meta.url))); + } catch { + return false; + } +} + +function resolveBackends(): Backend[] { + const raw = process.env.DKG_BENCH_STORE_BACKENDS?.trim(); + if (!raw) { + if (workerArtifactAvailable()) return ['inprocess', 'worker']; + // Loud, because the worker backend is the one this regression is about — a + // silent inprocess-only run would look like it measured the production path. + console.warn( + '[store-read-latency] worker backend SKIPPED (compiled adapter missing) — ' + + 'measuring `inprocess` idle read latency ONLY, NOT the production write-contention path. ' + + 'Run `pnpm --filter @origintrail-official/dkg-storage build` (or `pnpm bench:store-read`, ' + + 'which builds it) to include the worker backend.', + ); + return ['inprocess']; + } + const known = new Set(['inprocess', 'worker']); + const requested = raw.split(',').map((p) => p.trim().toLowerCase()).filter(Boolean); + for (const b of requested) { + if (!known.has(b as Backend)) { + throw new Error(`Unknown DKG_BENCH_STORE_BACKENDS entry "${b}". Expected: inprocess, worker`); + } + } + if (requested.includes('worker') && !workerArtifactAvailable()) { + throw new Error( + 'DKG_BENCH_STORE_BACKENDS requested "worker", but the compiled adapter is missing. ' + + 'Run `pnpm --filter @origintrail-official/dkg-storage build` first.', + ); + } + return requested.length > 0 ? (requested as Backend[]) : ['inprocess']; +} + function resolveStoreSizeLabels(): string[] { const raw = process.env.DKG_BENCH_STORE_SIZES?.trim(); const labels = raw ? raw.split(',').map((p) => p.trim().toLowerCase()).filter(Boolean) : ['1k', '50k']; @@ -90,8 +162,15 @@ function resolveStoreSizeLabels(): string[] { return labels; } +function createStore(backend: Backend): ReadStore { + // `worker` availability is gated in resolveBackends(), so by the time we get + // here the compiled adapter is present. + return backend === 'worker' ? new OxigraphWorkerStore() : new OxigraphStore(); +} + export default defineSuite({ params: { + backend: resolveBackends(), storeSize: resolveStoreSizeLabels(), }, baseline: { @@ -106,20 +185,63 @@ export default defineSuite({ warmup: 1, }, async setup(scene) { + const backend = scene.params.backend as Backend; const sizeLabel = scene.params.storeSize as string; const quadCount = STORE_SIZES[sizeLabel]; - const store: ReadStore = new OxigraphStore(); + const store = createStore(backend); - // Registered up-front so the store is still closed even if population or a - // benchmark case throws. + const churn = makeQuads(CHURN_BATCH, CHURN_OFFSET); + let writerActive = false; + let writerDone: Promise | undefined; + let writerError: unknown; + + const stopWriter = async (): Promise => { + if (!writerDone) return; + writerActive = false; + const done = writerDone; + writerDone = undefined; + // The loop captures its own failures into `writerError`, so this never rejects. + await done; + // Remove the churn batch so a half-applied cycle (an insert without its + // matching delete) can't leak into the next iteration's getTotalTriples count. + try { + await store.delete(churn); + } catch { + /* store may be mid-teardown */ + } + // Fail fast if the writer died: a stopped writer must never let an + // `(under write load)` case record a successful (effectively idle) sample. + if (writerError !== undefined) { + const err = writerError; + writerError = undefined; + throw new Error(`background writer died during the under-load iteration: ${errorText(err)}`); + } + }; + + // ONE ordered teardown for the scene: stop the writer and await its loop + // BEFORE closing the store, so an in-flight insert/delete can never race a + // closed store/worker. esbench may run scene teardown hooks concurrently, so + // all ordering lives inside this single callback. Registered up-front so the + // store is still closed (and any worker thread terminated) even if the + // population below throws. scene.teardown(async () => { + let stopErr: unknown; + try { + await stopWriter(); + } catch (err) { + stopErr = err; // capture; still close the store below + } + // `close()` is the only place the worker thread is joined, so a failure + // here is a real teardown bug — surface it (log + reject) rather than + // swallow it and let a broken worker benchmark look green. try { await store.close(); } catch (closeErr) { - console.error(`[store-read-latency] store.close() failed: ${errorText(closeErr)}`); + console.error(`[store-read-latency] store.close() failed (worker thread may not have joined): ${errorText(closeErr)}`); throw closeErr; } + if (stopErr !== undefined) throw stopErr; }); // Pre-populate the base graph the reads scan. @@ -134,5 +256,63 @@ export default defineSuite({ benchAsyncWithHooks(scene, 'read getTotalTriples (idle)', async () => { assertCountAtLeast(await store.query(READ_TOTAL_TRIPLES), quadCount, 'read getTotalTriples'); }, {}); + + // Contention is meaningful only on the worker backend (see file docstring): + // the in-process store is single-threaded + synchronous, so reads and writes + // cannot truly overlap. Skip the `(under write load)` cases for it rather + // than report a misleading same-thread number. + if (backend !== 'worker') return; + + // Background writer for the worker-backend `(under write load)` cases, + // scoped to each loaded iteration (not to case-execution order): + // - `beforeIteration` (startWriter) AWAITS one full insert/delete cycle so + // writes are provably queued in the worker before the measured read. + // - a writer that dies is recorded in `writerError` and surfaced as a + // FAILED case — startWriter throws if it died before the first cycle, + // the workload throws if it died mid-measurement, and stopWriter throws + // if it died just after — so a broken writer can never silently degrade + // into an idle read. + // - `afterIteration` (stopWriter) stops it and clears the churn batch. + const startWriter = async (): Promise => { + if (writerActive) return; + writerActive = true; + writerError = undefined; + let signalFirstCycle!: () => void; + const firstCycle = new Promise((resolve) => { signalFirstCycle = resolve; }); + writerDone = (async () => { + try { + let signalled = false; + while (writerActive) { + await store.insert(churn); + await store.delete(churn); + if (!signalled) { signalled = true; signalFirstCycle(); } + } + } catch (err) { + writerError = err; + } finally { + // Unblock the barrier even if the first cycle threw, so a writer + // failure surfaces (below) instead of hanging the benchmark. + signalFirstCycle(); + } + })(); + await firstCycle; + if (writerError !== undefined) { + throw new Error(`background writer failed before its first write cycle: ${errorText(writerError)}`); + } + }; + + benchAsyncWithHooks(scene, 'read LIMIT 1 (under write load)', async () => { + assertNonEmptySelect(await store.query(READ_LIMIT1), 'read LIMIT 1'); + if (writerError !== undefined) { + throw new Error(`background writer died during the measurement: ${errorText(writerError)}`); + } + }, { beforeIteration: startWriter, afterIteration: stopWriter }); + + benchAsyncWithHooks(scene, 'read getTotalTriples (under write load)', async () => { + assertCountAtLeast(await store.query(READ_TOTAL_TRIPLES), quadCount, 'read getTotalTriples'); + if (writerError !== undefined) { + throw new Error(`background writer died during the measurement: ${errorText(writerError)}`); + } + }, { beforeIteration: startWriter, afterIteration: stopWriter }); }, }); diff --git a/docs/use-dkg/storage-sparql-http.md b/docs/use-dkg/storage-sparql-http.md index 3a8ea2c5b8..9d70af2f93 100644 --- a/docs/use-dkg/storage-sparql-http.md +++ b/docs/use-dkg/storage-sparql-http.md @@ -7,7 +7,7 @@ doc_type: how-to # Using an external SPARQL store (Oxigraph server, etc.) -The DKG node can use any **SPARQL 1.1 Protocol**–compliant store you run yourself, instead of its default daemon-managed local Oxigraph server. That gives you: +The DKG node can use any **SPARQL 1.1 Protocol**–compliant store you run yourself, instead of its default daemon-managed local Oxigraph server (or the embedded `oxigraph-worker` fallback). That gives you: - **Real on-disk persistence** (e.g. Oxigraph server with RocksDB) - **Larger graphs** without holding everything in the Node process @@ -118,4 +118,4 @@ await agent.start(); New installs default to a **daemon-managed local Oxigraph server** (`store.backend: "oxigraph-server"`): `dkg init`, `dkg openclaw/hermes/mcp setup`, or accepting the wizard default writes this block. The daemon fetches the pinned `oxigraph` binary on first boot and runs it on loopback, giving MVCC concurrent reads and incremental RocksDB persistence. -If a config has **no** `store` block at all, the runtime now uses the same daemon-managed **`oxigraph-server`** default. The old embedded **`oxigraph-worker`** backend has been retired; configs that still name it fail fast with a migration message. For very large graphs or existing infrastructure, use `sparql-http` with an external store. +If a config has **no** `store` block at all, the runtime falls back to the embedded in-process **`oxigraph-worker`** (a single-writer store that rewrites its on-disk N-Quads dump under `dataDir` on every flush) — fine for development and small nodes. For very large graphs or existing infrastructure, use `sparql-http` with an external store. diff --git a/packages/agent/src/dkg-agent-helpers.ts b/packages/agent/src/dkg-agent-helpers.ts index 5e7a39b814..3d53b4157a 100644 --- a/packages/agent/src/dkg-agent-helpers.ts +++ b/packages/agent/src/dkg-agent-helpers.ts @@ -403,6 +403,7 @@ export function applyDefaultLargeLiteralStorage( export function isLocalOxigraphConfig(storeConfig: TripleStoreConfig): boolean { return storeConfig.backend === 'oxigraph' + || storeConfig.backend === 'oxigraph-worker' || storeConfig.backend === 'oxigraph-persistent'; } diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index a64cd160a7..7b3825d16e 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -923,7 +923,7 @@ export interface DKGAgentConfig { }>; dataDir?: string; store?: TripleStore; - /** Triple store backend configuration (e.g. oxigraph-server runtime view, oxigraph-persistent, blazegraph). If omitted, dataDir agents use oxigraph-persistent. */ + /** Triple store backend configuration (e.g. oxigraph-worker, blazegraph). If omitted, defaults to oxigraph-worker when dataDir is set. */ storeConfig?: TripleStoreConfig; /** Out-of-line storage for large public SWM RDF literal object terms. Defaults on for local Oxigraph-backed dataDir stores. */ largeLiteralStorage?: LargeLiteralStorageConfig; diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 7d360f6c8b..f27f95994a 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -685,11 +685,11 @@ export class DKGAgent extends DKGAgentBase { const { join } = await import('node:path'); const persistPath = join(config.dataDir, 'store.nq'); store = await createTripleStore({ - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', options: { path: persistPath }, largeLiteralStorage: defaultLargeLiteralStorage(config.dataDir, config.largeLiteralStorage), }); - log.info(ctx, `Persistent triple store: ${persistPath}`); + log.info(ctx, `Persistent triple store (worker thread): ${persistPath}`); } else { store = await createTripleStore({ backend: 'oxigraph' }); log.warn(ctx, `No dataDir — triple store is in-memory (data will be lost on restart)`); diff --git a/packages/agent/src/sync/responder/sync-handler.ts b/packages/agent/src/sync/responder/sync-handler.ts index 1ea7142003..646afe3515 100644 --- a/packages/agent/src/sync/responder/sync-handler.ts +++ b/packages/agent/src/sync/responder/sync-handler.ts @@ -447,7 +447,7 @@ export function registerSyncHandler(params: RegisterSyncHandlerParams): void { warnedPreDispatchCancellation = true; logWarn( createOperationContext('sync'), - 'Sync responder is using a store backend whose query AbortSignal is pre-dispatch only; in-flight sync queries cannot release responder capacity until the synchronous store call returns. Use an HTTP SPARQL backend for interruptible long-query cancellation.', + 'Sync responder is using a store backend whose query AbortSignal is pre-dispatch only; in-flight sync queries cannot release responder capacity until the synchronous store call returns. Use oxigraph-worker or an HTTP SPARQL backend for interruptible long-query cancellation.', ); } if (isWorkspace) { diff --git a/packages/agent/test/context-graph-discovery.test.ts b/packages/agent/test/context-graph-discovery.test.ts index 33429c681e..bda11d560d 100644 --- a/packages/agent/test/context-graph-discovery.test.ts +++ b/packages/agent/test/context-graph-discovery.test.ts @@ -1303,10 +1303,8 @@ describe('listContextGraphs merge', () => { }, 15000); it('bypasses list cache for unknown configured store backends', async () => { - const backend = registerTripleStoreAdapter( - 'test-remote-list-cache-backend', - async () => new OxigraphStore(), - ); + const backend = 'test-remote-list-cache-backend'; + registerTripleStoreAdapter(backend, async () => new OxigraphStore()); const created = await DKGAgent.create({ kaNumberAllocator: makeTestKaNumberAllocator(), name: 'ContextGraphTestAgent', diff --git a/packages/cli/src/commands/hermes.ts b/packages/cli/src/commands/hermes.ts index 7f98eb0359..1d95782816 100644 --- a/packages/cli/src/commands/hermes.ts +++ b/packages/cli/src/commands/hermes.ts @@ -32,7 +32,6 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; -import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -190,7 +189,7 @@ hermesCmd ) .option( '--store ', - `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL and persists the store block after setup.`, + 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL and persists the store block after setup.', ) .option( '--store-url ', diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index d49ad22b9b..4bd216b3fe 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -36,7 +36,6 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; -import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -217,7 +216,7 @@ program ) .option( '--store ', - `Pre-fill the triple-store backend prompt (${storeFlagBackendList(' | ')}).`, + 'Pre-fill the triple-store backend prompt (oxigraph | blazegraph | sparql-http).', ) .option( '--store-url ', @@ -535,8 +534,9 @@ program chain: isNetworkSwitch ? chainSection : (chainSection ?? existing.chain), auth: { enabled: enableAuth, tokens: existing.auth?.tokens }, // Persist the chosen backend. `storeBlock === null` from the - // wizard means "leave the store block omitted"; daemon boot treats - // that as the managed `oxigraph-server` default. + // wizard means "use the local default" — we explicitly clear any + // existing block so re-running `dkg init` to switch from + // blazegraph back to oxigraph actually applies. store: storeBlock ?? undefined, }; await saveConfig(config); @@ -573,7 +573,7 @@ program const endpoint = o?.url ?? o?.queryEndpoint; return `${storeBlock.backend}${endpoint ? ` (${endpoint})` : ''}`; })() - : 'oxigraph-server (default)' + : 'oxigraph (local default)' }`, ); { diff --git a/packages/cli/src/commands/lifecycle.ts b/packages/cli/src/commands/lifecycle.ts index c4c8723c68..5dace2f9ba 100644 --- a/packages/cli/src/commands/lifecycle.ts +++ b/packages/cli/src/commands/lifecycle.ts @@ -290,7 +290,7 @@ program // remote store. Quad count = null is rare in practice (cached // every 30 s on the daemon side) so when it shows up the // operator should treat it as an alert, not a no-op. - const backend = s.storeBackend ?? 'oxigraph-server'; + const backend = s.storeBackend ?? 'oxigraph-worker'; if (s.storeUrl) { const quads = s.storeQuads == null ? 'UNREACHABLE' : `${s.storeQuads.toLocaleString()} quads`; console.log(` Store: ${backend} (${s.storeUrl}) — ${quads}`); diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts index d11ad64eaf..9ff4c39d9c 100644 --- a/packages/cli/src/commands/mcp.ts +++ b/packages/cli/src/commands/mcp.ts @@ -29,7 +29,6 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; -import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -158,7 +157,7 @@ mcpCmd .option('--yes', 'Auto-confirm per-client registrations (default false: prompt interactively in TTY mode; non-TTY auto-confirms — pass `--yes` in scripts for the safer scripted-environment posture)') .option( '--store ', - `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL and persists the store block after setup.`, + 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL and persists the store block after setup.', ) .option( '--store-url ', diff --git a/packages/cli/src/commands/openclaw.ts b/packages/cli/src/commands/openclaw.ts index d0fa61b47c..1d23846f7e 100644 --- a/packages/cli/src/commands/openclaw.ts +++ b/packages/cli/src/commands/openclaw.ts @@ -32,7 +32,6 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; -import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -128,7 +127,7 @@ openclawCmd ) .option( '--store ', - `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL via an ASK probe and persists the store block after setup completes.`, + 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL via an ASK probe and persists the store block after setup completes.', ) .option( '--store-url ', diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index dd2baa784a..6e4fa09152 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -19,12 +19,6 @@ import { STORAGE_ACK_TIMING_SAFETY_MARGIN_MS, type StorageAckTiming, } from '@origintrail-official/dkg-publisher'; -import { - getStoreBackendPolicy, - isExternalStoreBackend, - isRetiredStoreBackend, - configBackendList, -} from './store-backends.js'; /** * Per-step build timeouts (milliseconds) used by the git-based auto-update @@ -580,7 +574,7 @@ export interface DkgConfig { llm?: LlmConfig; /** Block explorer URL for TX links (default: derived from chainId). */ blockExplorerUrl?: string; - /** Triple store backend override (default: daemon-managed oxigraph-server). */ + /** Triple store backend override (default: oxigraph-worker with file persistence). */ store?: { backend: string; options?: Record; graphSetIndex?: boolean | GraphSetIndexConfig; changelog?: boolean }; /** * Intentional cap on how many persisted context-graph subscriptions a node @@ -1837,31 +1831,34 @@ export interface StoreConfigValidationError { export function validateStoreConfig(config: DkgConfig): StoreConfigValidationError[] { const errors: StoreConfigValidationError[] = []; const backend = config.store?.backend; - if (isRetiredStoreBackend(backend)) { - return [{ - field: 'store.backend', - message: - `${EXTERNAL_VALIDATION_PREFIX} "${backend}" is no longer supported. ` + - `Use one of: ${configBackendList()}.`, - }]; - } - if (!isExternalStoreBackend(backend)) return errors; + // Mirror of `isExternalBackend` from @origintrail-official/dkg-storage. + // Duplicated here to keep config.ts free of upward dependencies on the + // storage package (config.ts is leaf-imported by many other modules). + const isExternal = backend === 'blazegraph' || backend === 'sparql-http'; + if (!isExternal) return errors; const opts = (config.store?.options ?? {}) as Record; - const policy = getStoreBackendPolicy(backend); - if (!policy || policy.kind !== 'external') { - throw new Error(`Missing external-store policy for "${backend}"`); - } - const queryOption = policy.queryEndpointOption; - const queryEndpoint = opts[queryOption]; - if (typeof queryEndpoint !== 'string' || !queryEndpoint.trim()) { - errors.push({ - field: `store.options.${queryOption}`, - message: - `${EXTERNAL_VALIDATION_PREFIX} is "${backend}" but ` + - `store.options.${queryOption} is missing. Set it to the SPARQL query endpoint URL ` + - `or switch backend to oxigraph-server.`, - }); + + if (backend === 'blazegraph') { + if (typeof opts.url !== 'string' || !opts.url.trim()) { + errors.push({ + field: 'store.options.url', + message: + `${EXTERNAL_VALIDATION_PREFIX} is "blazegraph" but ` + + `store.options.url is missing. Set it to the SPARQL endpoint URL ` + + `(e.g. http://127.0.0.1:9999/bigdata/namespace/mynode/sparql) or ` + + `switch backend to oxigraph-worker.`, + }); + } + } else if (backend === 'sparql-http') { + if (typeof opts.queryEndpoint !== 'string' || !opts.queryEndpoint.trim()) { + errors.push({ + field: 'store.options.queryEndpoint', + message: + `${EXTERNAL_VALIDATION_PREFIX} is "sparql-http" but ` + + `store.options.queryEndpoint is missing. Set it to the SPARQL query URL.`, + }); + } } if (config.largeLiteralStorage?.enabled === true) { diff --git a/packages/cli/src/daemon/chain-reset-wipe.ts b/packages/cli/src/daemon/chain-reset-wipe.ts index 6d56c8bde4..2bb6df3a2c 100644 --- a/packages/cli/src/daemon/chain-reset-wipe.ts +++ b/packages/cli/src/daemon/chain-reset-wipe.ts @@ -64,6 +64,8 @@ */ import { existsSync, + readFileSync, + writeFileSync, readdirSync, rmSync, renameSync, @@ -72,14 +74,32 @@ import { } from 'node:fs'; import { join } from 'node:path'; import { isExternalBackend, getSparqlEndpoint, CHANGELOG_GRAPH } from '@origintrail-official/dkg-storage'; -import { - readPersistedDaemonState, - readPersistedNetworkConfig, - readPersistedStoreBackend, - writePersistedChainResetMarker, - writePersistedNetworkConfig, - writePersistedStoreBackend, -} from './daemon-state.js'; + +const STATE_FILE = '.network-state.json'; + +interface PersistedNetworkState { + /** Last chainResetMarker value the daemon booted on. */ + chainResetMarker: string | null; + /** + * Last triple-store backend the daemon booted on. Used by + * `detectBackendSwitch` to warn loudly when an operator hand-edits + * `config.store.backend` between boots — the new backend is fresh + * and empty, so silently booting would mean stale SWM/VM data is + * inaccessible. `null` on legacy state files (pre-RFC 120) and on + * first boot. (RFC 120 review point #6.) + */ + lastBackend?: string | null; + /** + * Last resolved network the daemon booted on (the `networkConfig` overlay + * name, e.g. `mainnet-gnosis`/`testnet`). Used by `detectNetworkSwitch` to + * abort boot when an operator repoints `config.networkConfig` at a different + * network on an existing data dir — the store holds the old network's + * chain-derived state (KC ids, merkle roots), which is meaningless on the + * new chain. `null`/absent on legacy state files and on first boot. + */ + lastNetworkConfig?: string | null; + savedAt: number; +} /** * Subset of `DkgConfig['store']` used by the wipe step to talk to an @@ -209,6 +229,69 @@ export function skipChainResetWipe(env: NodeJS.ProcessEnv = process.env): boolea return env.DKG_SKIP_CHAIN_RESET_WIPE === '1'; } +function loadState(dataDir: string): PersistedNetworkState | null { + try { + const raw = readFileSync(join(dataDir, STATE_FILE), 'utf8'); + const obj = JSON.parse(raw) as PersistedNetworkState; + if (typeof obj?.chainResetMarker !== 'string' && obj?.chainResetMarker !== null) return null; + return obj; + } catch { + return null; + } +} + +function saveState(dataDir: string, marker: string | null): void { + // Preserve any sibling fields (lastBackend) that `detectBackendSwitch` + // may have written. Otherwise a chain-reset wipe would clobber a + // freshly-recorded backend tag and the next boot would re-warn. + const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; + writeFileSync( + join(dataDir, STATE_FILE), + JSON.stringify( + { + ...existing, + chainResetMarker: marker, + savedAt: Date.now(), + } satisfies PersistedNetworkState, + null, + 2, + ), + ); +} + +function saveBackendTag(dataDir: string, backend: string): void { + const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; + writeFileSync( + join(dataDir, STATE_FILE), + JSON.stringify( + { + ...existing, + lastBackend: backend, + savedAt: Date.now(), + } satisfies PersistedNetworkState, + null, + 2, + ), + ); +} + +function saveNetworkTag(dataDir: string, networkConfig: string): void { + // Preserve sibling fields (chainResetMarker, lastBackend) like saveBackendTag. + const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; + writeFileSync( + join(dataDir, STATE_FILE), + JSON.stringify( + { + ...existing, + lastNetworkConfig: networkConfig, + savedAt: Date.now(), + } satisfies PersistedNetworkState, + null, + 2, + ), + ); +} + /** * Wipe the V10 data sitting in an external SPARQL endpoint. Runs after * the local file wipe so we don't strand the operator with a wiped FS @@ -499,7 +582,7 @@ export async function chainResetWipe( return { wiped: false, skipped: false, prevMarker: null, removedFiles: [], backedUpFiles: [], failedFiles: [] }; } - const prev = readPersistedDaemonState(opts.dataDir); + const prev = loadState(opts.dataDir); const prevMarker = prev?.chainResetMarker ?? null; if (prevMarker === opts.currentMarker) { @@ -581,7 +664,7 @@ export async function chainResetWipe( if (failedFiles.length === 0) { try { - writePersistedChainResetMarker(opts.dataDir, opts.currentMarker); + saveState(opts.dataDir, opts.currentMarker); markerPersisted = true; } catch (err) { log( @@ -635,7 +718,7 @@ export interface BackendSwitchDetectOptions { /** * Backend name from the current config. Pass the effective value * including the default — e.g. when `config.store?.backend` is - * undefined, callers should pass `'oxigraph-server'` so the check + * undefined, callers should pass `'oxigraph-worker'` so the check * is symmetric across "no store block" ↔ "explicit store block". */ currentBackend: string; @@ -667,17 +750,21 @@ export function detectBackendSwitch( opts: BackendSwitchDetectOptions, ): BackendSwitchDetectResult { const log = opts.log ?? (() => {}); - const previous = readPersistedStoreBackend(opts.dataDir); + const prev = loadState(opts.dataDir); + const previous = + typeof prev?.lastBackend === 'string' && prev.lastBackend.length > 0 + ? prev.lastBackend + : null; // First boot or legacy state file: silently record and move on. We - // explicitly do NOT treat null-previous as a "switch from the old - // implicit backend"; that would re-warn every operator who upgrades + // explicitly do NOT treat null-previous as a "switch from + // oxigraph-worker"; that would re-warn every operator who upgrades // into this release without ever having touched their store // configuration. Only operator-visible config changes between two // recorded backends count as a switch. if (previous === null) { try { - writePersistedStoreBackend(opts.dataDir, opts.currentBackend); + saveBackendTag(opts.dataDir, opts.currentBackend); } catch { // Non-fatal: if we can't write the tag now, we'll try again next // boot. The downside is one missed early-warning window. @@ -714,7 +801,7 @@ export function detectBackendSwitch( log(``); log(`DKG_ACCEPT_STORE_RESET=1 set — proceeding with the new backend.`); try { - writePersistedStoreBackend(opts.dataDir, opts.currentBackend); + saveBackendTag(opts.dataDir, opts.currentBackend); } catch (err) { log(`WARN: failed to persist new backend tag: ${(err as Error).message}. Will re-warn on next boot.`); } @@ -764,7 +851,11 @@ export function detectNetworkSwitch( opts: NetworkSwitchDetectOptions, ): NetworkSwitchDetectResult { const log = opts.log ?? (() => {}); - const previous = readPersistedNetworkConfig(opts.dataDir); + const prev = loadState(opts.dataDir); + const previous = + typeof prev?.lastNetworkConfig === 'string' && prev.lastNetworkConfig.length > 0 + ? prev.lastNetworkConfig + : null; // First boot or legacy state file: silently record and move on. We do NOT // treat null-previous as a switch — that would abort every operator who @@ -775,7 +866,7 @@ export function detectNetworkSwitch( // are caught normally. if (previous === null) { try { - writePersistedNetworkConfig(opts.dataDir, opts.currentNetworkConfig); + saveNetworkTag(opts.dataDir, opts.currentNetworkConfig); } catch { // Non-fatal: retry the tag write next boot. } @@ -815,7 +906,7 @@ export function detectNetworkSwitch( log(``); log(`DKG_ACCEPT_NETWORK_SWITCH=1 set — proceeding on the new network.`); try { - writePersistedNetworkConfig(opts.dataDir, opts.currentNetworkConfig); + saveNetworkTag(opts.dataDir, opts.currentNetworkConfig); } catch (err) { log(`WARN: failed to persist new network tag: ${(err as Error).message}. Will re-warn on next boot.`); } diff --git a/packages/cli/src/daemon/daemon-state.ts b/packages/cli/src/daemon/daemon-state.ts deleted file mode 100644 index 66d546821e..0000000000 --- a/packages/cli/src/daemon/daemon-state.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; - -export const DAEMON_STATE_FILE = '.network-state.json'; - -/** Persisted boot state shared by reset and configuration-switch guards. */ -export interface PersistedDaemonState { - chainResetMarker: string | null; - lastBackend?: string | null; - lastNetworkConfig?: string | null; - savedAt: number; -} - -export function readPersistedDaemonState(dataDir: string): PersistedDaemonState | null { - try { - const raw = readFileSync(join(dataDir, DAEMON_STATE_FILE), 'utf8'); - const state = JSON.parse(raw) as PersistedDaemonState; - if ( - typeof state?.chainResetMarker !== 'string' - && state?.chainResetMarker !== null - ) { - return null; - } - return state; - } catch { - return null; - } -} - -function updatePersistedDaemonState( - dataDir: string, - patch: Partial>, -): void { - const existing = readPersistedDaemonState(dataDir) ?? { - chainResetMarker: null, - savedAt: 0, - }; - writeFileSync( - join(dataDir, DAEMON_STATE_FILE), - JSON.stringify( - { - ...existing, - ...patch, - savedAt: Date.now(), - } satisfies PersistedDaemonState, - null, - 2, - ), - ); -} - -export function readPersistedStoreBackend(dataDir: string): string | null { - const state = readPersistedDaemonState(dataDir); - return typeof state?.lastBackend === 'string' && state.lastBackend.length > 0 - ? state.lastBackend - : null; -} - -export function readPersistedNetworkConfig(dataDir: string): string | null { - const state = readPersistedDaemonState(dataDir); - return typeof state?.lastNetworkConfig === 'string' && state.lastNetworkConfig.length > 0 - ? state.lastNetworkConfig - : null; -} - -export function writePersistedChainResetMarker(dataDir: string, marker: string | null): void { - updatePersistedDaemonState(dataDir, { chainResetMarker: marker }); -} - -export function writePersistedStoreBackend(dataDir: string, backend: string): void { - updatePersistedDaemonState(dataDir, { lastBackend: backend }); -} - -export function writePersistedNetworkConfig(dataDir: string, networkConfig: string): void { - updatePersistedDaemonState(dataDir, { lastNetworkConfig: networkConfig }); -} diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index 918080a9f8..4297ac48c3 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -311,12 +311,7 @@ import { reverseLocalAgentSetupForUi, refreshLocalAgentIntegrationFromUi, } from './local-agents.js'; -import { - createRequestStoreContext, - type MemoryGraphChangedEvent, - type NotificationSseEvent, - type RequestContext, -} from './routes/context.js'; +import type { MemoryGraphChangedEvent, NotificationSseEvent, RequestContext } from './routes/context.js'; import { handleStatusRoutes } from './routes/status.js'; import { handleAgentChatRoutes } from './routes/agent-chat.js'; import { handleOpenclawRoutes } from './routes/openclaw.js'; @@ -335,7 +330,6 @@ import { handleOperationalWalletRoutes } from './routes/operational-wallets.js'; import { handleNotificationRoutes } from './routes/notifications.js'; import { handlePluginRoutes } from './routes/plugins.js'; import type { RoutePlugin } from './plugin-api.js'; -import type { StoreRuntimeContext } from './store-runtime.js'; export async function handleRequest( @@ -344,7 +338,7 @@ export async function handleRequest( agent: DKGAgent, publisherControl: ReturnType, publisherRuntime: PublisherRuntime | null, - storeRuntime: StoreRuntimeContext, + config: DkgConfig, startedAt: number, dashDb: DashboardDB, opWallets: import("@origintrail-official/dkg-agent").OpWalletsConfig, @@ -389,7 +383,7 @@ export async function handleRequest( publisherControl, publisherRuntime, publisherAvailability, - ...createRequestStoreContext(storeRuntime), + config, startedAt, dashDb, opWallets, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 1fa6438cac..6aad669619 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -40,7 +40,7 @@ import { import { execSync, exec, execFile } from "node:child_process"; import { promisify } from "node:util"; import { join, dirname, resolve } from 'node:path'; -import { readdirSync, readFileSync, openSync, closeSync, writeFileSync as fsWriteFileSync, unlinkSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync, openSync, closeSync, writeFileSync as fsWriteFileSync, unlinkSync } from 'node:fs'; // Namespace import: our Phase-8 install-context builder (~line 290) calls // `osModule.homedir()`, and the later agent-identity probe (~line 6851) // uses `osModule.hostname()` + `osModule.userInfo()`. v10-rc's new @@ -306,10 +306,6 @@ import { } from './store-health-check.js'; import { startManagedOxigraph } from './oxigraph-managed.js'; import type { OxigraphServerHandle } from './oxigraph-server.js'; -import { - resolveDaemonStoreBootPlan, - resolveDaemonStoreRuntime, -} from './store-runtime.js'; import { resetNatStatus, startNatStatusWatcher } from './nat-status.js'; import { OPENCLAW_UI_CONNECT_TIMEOUT_MS, @@ -1131,25 +1127,6 @@ export async function runDaemonInner( ...resolveNetworkDefaultContextGraphs(network), ]), ]; - const acceptStoreReset = process.env.DKG_ACCEPT_STORE_RESET === '1'; - const storeDecision = resolveDaemonStoreBootPlan({ - config, - dataDir: dkgDir(), - acceptStoreReset, - }); - - if (storeDecision.kind === 'invalid-config') { - exitOnStoreConfigErrors(storeDecision.operatorConfig, log); - throw new Error('Invalid store config validation unexpectedly returned'); - } - if (storeDecision.kind === 'blocked-legacy-cutover') { - log(storeDecision.message); - process.exit(1); - } - const storeBoot = storeDecision; - if (storeBoot.notice) { - log(storeBoot.notice); - } // Auto-wipe per-node chain-state derived files (oxigraph store, publish // journal, random-sampling WAL) when the maintainer bumps @@ -1166,8 +1143,8 @@ export async function runDaemonInner( // silently would look like data loss to the operator. const backendSwitch = detectBackendSwitch({ dataDir: dkgDir(), - currentBackend: storeBoot.effectiveStore.backend, - acceptStoreReset, + currentBackend: config.store?.backend ?? 'oxigraph-worker', + acceptStoreReset: process.env.DKG_ACCEPT_STORE_RESET === '1', log, }); if (backendSwitch.aborted) { @@ -1203,7 +1180,7 @@ export async function runDaemonInner( let managedOxigraph: OxigraphServerHandle | null = null; let managed: Awaited> = null; try { - managed = await startManagedOxigraph({ config: storeBoot.effectiveConfig, dataDir: dkgDir(), log }); + managed = await startManagedOxigraph({ config, dataDir: dkgDir(), log }); if (managed) { managedOxigraph = managed.handle; // Every remaining fatal boot path (config validation, store health @@ -1216,7 +1193,7 @@ export async function runDaemonInner( } catch (err) { log( `[STORE] failed to start managed Oxigraph server: ${(err as Error).message}\n` + - `Fix the cause, or switch \`store.backend\` to ` + + `Fix the cause, or switch \`store.backend\` to oxigraph-worker (embedded) or ` + `sparql-http (operator-managed endpoint) in ~/.dkg/config.json.`, ); process.exit(1); @@ -1234,13 +1211,22 @@ export async function runDaemonInner( // For the directory-backed blob/snapshot stores we use the managed // defaults (the rewritten sparql-http backend has no `options.path` to // infer a directory from, unlike the local Oxigraph backend). - const storeRuntime = resolveDaemonStoreRuntime(storeBoot, managed); - const { - runtimeStore, - runtimeConfig: runtimeStoreConfig, - runtimeLargeLiteralStorage, - runtimeSnapshotStorage, - } = storeRuntime; + const runtimeStore = managed?.storeConfig ?? config.store; + const runtimeLargeLiteralStorage = + managed?.largeLiteralStorage ?? config.largeLiteralStorage; + const runtimeSnapshotStorage = + managed?.sharedMemoryPublicSnapshotStorage ?? config.sharedMemoryPublicSnapshotStorage; + // Config view used only for the boot-time store validation/health steps + // below: same as `config` but with the runtime store/blob/snapshot values + // swapped in, so a managed config validates against what actually runs. + const runtimeStoreConfig: DkgConfig = managed + ? { + ...config, + store: runtimeStore, + largeLiteralStorage: runtimeLargeLiteralStorage, + sharedMemoryPublicSnapshotStorage: runtimeSnapshotStorage, + } + : config; // Refuse to start on invalid external-backend config (missing URL, // missing blob/snapshot directory). This fires before the health @@ -1993,7 +1979,7 @@ export async function runDaemonInner( try { const runtime = await startPublisherRuntimeIfEnabled({ dataDir: dkgDir(), - config: runtimeStoreConfig, + config, store: agent.store, keypair: agent.wallet.keypair, chainBase: publisherChainBase, @@ -3374,7 +3360,7 @@ export async function runDaemonInner( agent, publisherControl, publisherRuntime, - storeRuntime, + config, startedAt, dashDb, opWallets, diff --git a/packages/cli/src/daemon/oxigraph-managed.ts b/packages/cli/src/daemon/oxigraph-managed.ts index 948505a601..1dd6cd6b5f 100644 --- a/packages/cli/src/daemon/oxigraph-managed.ts +++ b/packages/cli/src/daemon/oxigraph-managed.ts @@ -23,7 +23,6 @@ * matching the Blazegraph-Docker provisioner's contract. */ import { join } from 'node:path'; -import { MANAGED_DAEMON_STORE_BACKEND } from '../store-backends.js'; import { ensureOxigraphBinary } from './oxigraph-binary.js'; import { startOxigraphServer, @@ -36,7 +35,7 @@ import { } from './oxigraph-launch-strategy.js'; /** Config value that opts a node into the daemon-managed local server. */ -export { MANAGED_DAEMON_STORE_BACKEND as MANAGED_OXIGRAPH_BACKEND }; +export const MANAGED_OXIGRAPH_BACKEND = 'oxigraph-server'; /** Default loopback bind port. Override via `store.options.port`. */ export const DEFAULT_OXIGRAPH_PORT = 7878; @@ -145,7 +144,7 @@ export function planManagedOxigraph( config: ConfigLike, dataDir: string, ): ManagedOxigraphPlan | null { - if (config.store?.backend !== MANAGED_DAEMON_STORE_BACKEND) return null; + if (config.store?.backend !== MANAGED_OXIGRAPH_BACKEND) return null; const options = config.store.options ?? {}; const port = resolveManagedOxigraphPort(options); diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index c29268c4b4..8b37447315 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -23,7 +23,6 @@ import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; import type { CatchupTracker } from '../types.js'; import type { RoutePlugin } from '../plugin-api.js'; import type { AdmissionStatsView } from '../http-utils.js'; -import type { StoreRuntimeContext } from '../store-runtime.js'; export type MemoryGraphLayer = 'wm' | 'swm' | 'vm'; @@ -55,29 +54,7 @@ export interface NotificationSseEvent { type: string; } -/** - * Store views exposed to routes. The operator config is intentionally the only - * config object in this shape, so a direct route harness cannot provide a - * second, contradictory operator config through a nested store context. - */ -export interface RequestStoreContext { - /** Operator config exactly as loaded from disk / CLI. */ - config: DkgConfig; - /** Daemon-facing backend after defaults and acknowledged migrations. */ - effectiveStore: StoreRuntimeContext['effectiveStore']; - /** Constructible live adapter config after managed-store materialization. */ - runtimeStore: StoreRuntimeContext['runtimeStore']; -} - -export function createRequestStoreContext(storeRuntime: StoreRuntimeContext): RequestStoreContext { - return { - config: storeRuntime.operatorConfig, - effectiveStore: storeRuntime.effectiveStore, - runtimeStore: storeRuntime.runtimeStore, - }; -} - -export interface RequestContext extends RequestStoreContext { +export interface RequestContext { req: IncomingMessage; res: ServerResponse; agent: DKGAgent; @@ -85,6 +62,7 @@ export interface RequestContext extends RequestStoreContext { publisherRuntime: PublisherRuntime | null; /** Lifecycle-owned publisher state; optional for direct route embeddings/tests. */ publisherAvailability?: AsyncPublisherAvailability; + config: DkgConfig; startedAt: number; dashDb: DashboardDB; opWallets: OpWalletsConfig; diff --git a/packages/cli/src/daemon/routes/status.ts b/packages/cli/src/daemon/routes/status.ts index aa052f979e..1ddf8eade3 100644 --- a/packages/cli/src/daemon/routes/status.ts +++ b/packages/cli/src/daemon/routes/status.ts @@ -57,10 +57,8 @@ const execAsync = promisify(exec); const execFileAsync = promisify(execFile); import { enrichEvmError, MockChainAdapter, resolveRpcUrls, getRpcFailoverStats } from '@origintrail-official/dkg-chain'; import { DKGAgent, loadOpWallets } from '@origintrail-official/dkg-agent'; -import { - isExternalStoreBackend as isExternalBackend, - isManagedLocalBackend, -} from '../../store-backends.js'; +import { isExternalBackend } from '@origintrail-official/dkg-storage'; +import { resolveManagedOxigraphPort } from '../oxigraph-managed.js'; import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri } from '@origintrail-official/dkg-core'; import { findReservedSubjectPrefix, isSkolemizedUri } from '@origintrail-official/dkg-publisher'; import { @@ -422,10 +420,6 @@ export function invalidateExternalStoreQuadsCache(): void { storeQuadsInflight = null; } -export function storeBackendHasStatusHealth(backend: string | undefined): boolean { - return isExternalBackend(backend) || isManagedLocalBackend(backend); -} - async function getCachedExternalStoreQuads( agent: DKGAgent, now: number, @@ -504,8 +498,6 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { agent, publisherControl, config, - effectiveStore, - runtimeStore, startedAt, dashDb, opWallets, @@ -659,7 +651,6 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { // sentinels when build-info.json is absent (monorepo / dev), // so consumers can branch reliably. const buildInfo = loadBuildInfo(); - const runtimeStoreOptions = (runtimeStore.options ?? {}) as Record; return jsonResponse(res, 200, { name: config.name, version: nodeVersion, @@ -675,25 +666,36 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { networkConfig: resolveNetworkConfigName(config), networkId, networkName: network?.networkName ?? null, - storeBackend: effectiveStore.backend, + storeBackend: config.store?.backend ?? "oxigraph-worker", // External backend visibility (RFC 120 / plan PR 1 item 3). For // local backends both fields stay null so the response shape is // stable across deployments. - storeUrl: isExternalBackend(runtimeStore.backend) + storeUrl: isExternalBackend(config.store?.backend) ? (() => { - const url = typeof runtimeStoreOptions.url === 'string' ? runtimeStoreOptions.url - : typeof runtimeStoreOptions.queryEndpoint === 'string' ? runtimeStoreOptions.queryEndpoint + const opts = (config.store?.options ?? {}) as Record; + const url = typeof opts.url === 'string' ? opts.url + : typeof opts.queryEndpoint === 'string' ? opts.queryEndpoint : null; return url; })() - : null, - // `storeBackend` describes effective daemon policy while URL and health - // come from the live constructible adapter. This matters for both the - // implicit managed default and acknowledged oxigraph-worker cutovers: - // operator config may be absent/retired, effective is oxigraph-server, - // and runtime is its materialized loopback sparql-http adapter. + : config.store?.backend === 'oxigraph-server' + // Managed local server: report its loopback endpoint so `dkg status` + // renders the external-store health path (storeQuads/unreachable) + // instead of printing it like a quad-less local store. + ? (() => { + const opts = (config.store?.options ?? {}) as Record; + const port = resolveManagedOxigraphPort(opts); + return `http://127.0.0.1:${port}/query`; + })() + : null, + // A managed `oxigraph-server` keeps `config.store.backend` as + // "oxigraph-server" (so it persists/labels correctly), but its quad + // count is still worth surfacing — it's the only store-health signal + // for that backend (getStoreBytes is null, there's no store.nq), and a + // failed query here is how operators see the managed server is down + // (e.g. after a failed revive) instead of it always looking healthy. storeQuads: - (storeBackendHasStatusHealth(effectiveStore.backend) || isExternalBackend(runtimeStore.backend)) + isExternalBackend(config.store?.backend) || config.store?.backend === 'oxigraph-server' ? await getCachedExternalStoreQuads(agent, Date.now()) : null, uptimeMs: Date.now() - startedAt, diff --git a/packages/cli/src/daemon/store-runtime.ts b/packages/cli/src/daemon/store-runtime.ts deleted file mode 100644 index 6f43d6dc15..0000000000 --- a/packages/cli/src/daemon/store-runtime.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; -import { - DEFAULT_DAEMON_STORE_BACKEND, - isManagedLocalBackend, - isRetiredStoreBackend, - requireStorageAdapterBackend, - type StorageAdapterBackend, -} from '../store-backends.js'; -import type { DkgConfig } from '../config.js'; -import { readPersistedStoreBackend } from './daemon-state.js'; -import type { ManagedOxigraphResult } from './oxigraph-managed.js'; - -type StoreConfig = NonNullable; -export type RuntimeStoreConfig = Omit & { - backend: StorageAdapterBackend; -}; - -interface DaemonStoreOperatorContext { - /** Operator-facing config exactly as loaded from disk / CLI. */ - operatorConfig: DkgConfig; -} - -export interface InvalidDaemonStoreConfig extends DaemonStoreOperatorContext { - kind: 'invalid-config'; -} - -export interface BlockedLegacyStoreCutover extends DaemonStoreOperatorContext { - kind: 'blocked-legacy-cutover'; - message: string; -} - -export interface DaemonStoreBootPlan extends DaemonStoreOperatorContext { - kind: 'bootable'; - /** Config with the implicit daemon default materialized for boot steps. */ - effectiveConfig: DkgConfig; - /** Store backend used for backend-switch detection and managed startup. */ - effectiveStore: StoreConfig; - /** Non-fatal startup notice, e.g. acknowledged legacy default cutover. */ - notice?: string; -} - -export type DaemonStoreBootDecision = - | InvalidDaemonStoreConfig - | BlockedLegacyStoreCutover - | DaemonStoreBootPlan; - -export interface DaemonStoreRuntimePlan extends DaemonStoreBootPlan { - /** Store config consumed by validation, health probes, wipe, and the agent. */ - runtimeStore: RuntimeStoreConfig; - /** Config view with runtime store/blob/snapshot values swapped in. */ - runtimeConfig: DkgConfig; - runtimeLargeLiteralStorage: DkgConfig['largeLiteralStorage']; - runtimeSnapshotStorage: DkgConfig['sharedMemoryPublicSnapshotStorage']; -} - -/** Explicit store views threaded into request routing after startup. */ -export interface StoreRuntimeContext { - /** Persisted/operator intent. Routes that save config must use this view. */ - operatorConfig: DkgConfig; - /** Daemon-facing backend after defaults and acknowledged migrations. */ - effectiveStore: StoreConfig; - /** Constructible live adapter config after managed-store materialization. */ - runtimeStore: RuntimeStoreConfig; -} - -export function resolveEffectiveDaemonStore(config: Pick): StoreConfig { - return config.store ?? { backend: DEFAULT_DAEMON_STORE_BACKEND, options: {} }; -} - -export function resolveDaemonStoreBootPlan(opts: { - config: DkgConfig; - dataDir: string; - acceptStoreReset: boolean; -}): DaemonStoreBootDecision { - const { config, dataDir, acceptStoreReset } = opts; - const legacyStorePath = join(dataDir, 'store.nq'); - const legacyStoreExists = existsSync(legacyStorePath); - const retiredStoreConfigured = isRetiredStoreBackend(config.store?.backend); - const configuredForManagedServer = !config.store || isManagedLocalBackend(config.store.backend); - const previousBackend = readPersistedStoreBackend(dataDir); - const legacyCutoverAlreadyRecorded = isManagedLocalBackend(previousBackend); - const legacyCutoverRequired = legacyStoreExists - && (configuredForManagedServer || retiredStoreConfigured) - && !legacyCutoverAlreadyRecorded; - const migrateAcknowledgedRetiredStore = retiredStoreConfigured - && legacyCutoverRequired - && acceptStoreReset; - const effectiveStore = migrateAcknowledgedRetiredStore - ? resolveEffectiveDaemonStore({}) - : resolveEffectiveDaemonStore(config); - const effectiveConfig = config.store && !migrateAcknowledgedRetiredStore - ? config - : { ...config, store: effectiveStore }; - - // A retired worker config with no legacy data has no migration decision to - // make. Keep that state separate so callers cannot accidentally continue to - // managed startup with an invalid operator config. - if (retiredStoreConfigured && !legacyCutoverRequired) { - return { kind: 'invalid-config', operatorConfig: config }; - } - - if (legacyCutoverRequired && !acceptStoreReset) { - const legacySource = retiredStoreConfigured - ? `${config.store?.backend} backend` - : config.store - ? 'worker-backed store' - : 'implicit worker default'; - return { - kind: 'blocked-legacy-cutover', - operatorConfig: config, - message: - `[STORE] oxigraph-worker support has been removed, but this node has a legacy ` + - `store.nq from the old ${legacySource}.\n` + - `Set store.backend to "oxigraph-server" (or an external SPARQL backend) and ` + - `restart with DKG_ACCEPT_STORE_RESET=1 to acknowledge the fresh-store cutover. ` + - `The legacy store.nq file is left untouched for manual backup or migration.`, - }; - } - - let notice: string | undefined; - if (legacyCutoverRequired && acceptStoreReset) { - notice = retiredStoreConfigured - ? `[STORE] explicit ${config.store?.backend} is retired; using oxigraph-server after reset acknowledgement. Legacy store.nq is left untouched.` - : '[STORE] using oxigraph-server after reset acknowledgement. Legacy store.nq is left untouched.'; - } else if (!config.store && acceptStoreReset) { - notice = - '[STORE] no store block found; using oxigraph-server. Legacy store.nq, if present, is left untouched.'; - } - - return { - kind: 'bootable', - operatorConfig: config, - effectiveConfig, - effectiveStore, - ...(notice ? { notice } : {}), - }; -} - -export function resolveDaemonStoreRuntime( - bootPlan: DaemonStoreBootPlan, - managed: ManagedOxigraphResult | null, -): DaemonStoreRuntimePlan { - if (isManagedLocalBackend(bootPlan.effectiveStore.backend) && !managed) { - throw new Error( - `Managed daemon store "${bootPlan.effectiveStore.backend}" was not materialized to a storage adapter`, - ); - } - const candidateStore = managed?.storeConfig ?? bootPlan.effectiveStore; - const runtimeStore: RuntimeStoreConfig = { - ...candidateStore, - backend: requireStorageAdapterBackend(candidateStore.backend), - }; - const runtimeLargeLiteralStorage = - managed?.largeLiteralStorage ?? bootPlan.operatorConfig.largeLiteralStorage; - const runtimeSnapshotStorage = - managed?.sharedMemoryPublicSnapshotStorage ?? bootPlan.operatorConfig.sharedMemoryPublicSnapshotStorage; - const runtimeConfig: DkgConfig = managed - ? { - ...bootPlan.effectiveConfig, - store: runtimeStore, - largeLiteralStorage: runtimeLargeLiteralStorage, - sharedMemoryPublicSnapshotStorage: runtimeSnapshotStorage, - } - : bootPlan.effectiveConfig; - - return { - ...bootPlan, - runtimeStore, - runtimeConfig, - runtimeLargeLiteralStorage, - runtimeSnapshotStorage, - }; -} diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index 0d07d43c27..adac992a45 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -6,7 +6,6 @@ import { ACKCollector, AsyncLiftRunner, DKGPublisher, FileWorkspacePublicSnapsho import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { loadNetworkConfig, resolveReadyChainConfig, type DkgConfig } from './config.js'; import { loadPublisherWallets } from './publisher-wallets.js'; -import { isManagedLocalBackend, isRetiredStoreBackend } from './store-backends.js'; export type { ACKTransportFactory } from '@origintrail-official/dkg-publisher'; @@ -589,18 +588,6 @@ function createChainRecoveryResolver( async function createPublisherStore(dataDir: string, config: DkgConfig): Promise { if (config.store) { - if (isManagedLocalBackend(config.store.backend)) { - throw new Error( - `Publisher commands for daemon-managed store "${config.store.backend}" require a running DKG daemon. ` + - 'Start the daemon and retry; daemon-down direct inspection cannot safely materialize the managed store.', - ); - } - if (isRetiredStoreBackend(config.store.backend)) { - throw new Error( - `Publisher commands cannot open retired store backend "${config.store.backend}" directly. ` + - 'Start the daemon to complete the acknowledged store migration, then retry.', - ); - } const storeConfig = config.store as any; return await createTripleStore({ ...storeConfig, @@ -619,7 +606,7 @@ async function createPublisherStore(dataDir: string, config: DkgConfig): Promise } return await createTripleStore({ - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', options: { path: join(dataDir, 'store.nq') }, largeLiteralStorage: defaultLargeLiteralStorage(dataDir, config), }); @@ -646,6 +633,7 @@ export function createPublicSnapshotStore( function isLocalOxigraphStoreConfig(storeConfig: { backend?: unknown }): boolean { return storeConfig.backend === 'oxigraph' + || storeConfig.backend === 'oxigraph-worker' || storeConfig.backend === 'oxigraph-persistent'; } diff --git a/packages/cli/src/store-backends.ts b/packages/cli/src/store-backends.ts deleted file mode 100644 index 87b78c8305..0000000000 --- a/packages/cli/src/store-backends.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { - STORAGE_ADAPTERS, - isExternalBackend, - isStorageAdapterBackend, - type StorageAdapterBackend, -} from '@origintrail-official/dkg-storage'; - -/** - * Operator-facing daemon policy composed on top of storage-owned adapter facts. - * - * This is the sole owner of daemon defaults, retired config names, migration - * classification, menu visibility, and labels. Adapter endpoint/path metadata - * is spread from `STORAGE_ADAPTERS` rather than duplicated here. - */ -export const STORE_BACKENDS = { - 'oxigraph-server': { - kind: 'managed-local', - adapter: false, - retired: false, - default: true, - wizard: true, - storeFlag: true, - label: 'oxigraph-server (managed local server — recommended)', - }, - oxigraph: { - ...STORAGE_ADAPTERS.oxigraph, - adapter: true, - retired: false, - default: false, - wizard: true, - storeFlag: true, - label: 'oxigraph (embedded in-memory store — development only)', - }, - 'oxigraph-persistent': { - ...STORAGE_ADAPTERS['oxigraph-persistent'], - adapter: true, - retired: false, - default: false, - wizard: false, - storeFlag: false, - }, - blazegraph: { - ...STORAGE_ADAPTERS.blazegraph, - adapter: true, - retired: false, - default: false, - wizard: true, - storeFlag: true, - label: 'blazegraph (external SPARQL endpoint)', - }, - 'sparql-http': { - ...STORAGE_ADAPTERS['sparql-http'], - adapter: true, - retired: false, - default: false, - wizard: false, - storeFlag: true, - }, - 'oxigraph-worker': { - kind: 'retired', - adapter: false, - retired: true, - default: false, - wizard: false, - storeFlag: false, - }, -} as const; - -export type StoreBackend = keyof typeof STORE_BACKENDS; -export type StoreBackendPolicy = (typeof STORE_BACKENDS)[StoreBackend]; -export type StoreBackendKind = StoreBackendPolicy['kind']; -export type StoreBackendOfKind = { - [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { kind: Kind } - ? Backend - : never; -}[StoreBackend]; -export type ConfigStoreBackend = { - [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { retired: false } - ? Backend - : never; -}[StoreBackend]; -export type RetiredStoreBackend = { - [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { retired: true } - ? Backend - : never; -}[StoreBackend]; -export type DefaultStoreBackend = { - [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { default: true } - ? Backend - : never; -}[StoreBackend]; -export type WizardStoreBackend = { - [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { wizard: true } - ? Backend - : never; -}[StoreBackend]; -export type StoreFlagBackend = { - [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { storeFlag: true } - ? Backend - : never; -}[StoreBackend]; -export type ExternalStoreBackend = Extract, StorageAdapterBackend>; -export type LocalStoreBackend = Extract, StorageAdapterBackend>; -export type ManagedLocalStoreBackend = StoreBackendOfKind<'managed-local'>; - -export function storeBackendNames(): StoreBackend[] { - return Object.keys(STORE_BACKENDS) as StoreBackend[]; -} - -function requireSingleBackend( - backends: readonly Backend[], - description: string, -): Backend { - if (backends.length !== 1) { - throw new Error(`Expected exactly one ${description} store backend, found ${backends.length}`); - } - return backends[0]; -} - -export const DEFAULT_DAEMON_STORE_BACKEND: DefaultStoreBackend = requireSingleBackend( - storeBackendNames().filter( - (backend): backend is DefaultStoreBackend => STORE_BACKENDS[backend].default, - ), - 'default', -); - -export const MANAGED_DAEMON_STORE_BACKEND: ManagedLocalStoreBackend = requireSingleBackend( - storeBackendNames().filter( - (backend): backend is ManagedLocalStoreBackend => STORE_BACKENDS[backend].kind === 'managed-local', - ), - 'managed-local', -); - -export const DEFAULT_STORE_BACKEND = DEFAULT_DAEMON_STORE_BACKEND; -export const MANAGED_LOCAL_STORE_BACKEND = MANAGED_DAEMON_STORE_BACKEND; - -export function configBackendNames(): ConfigStoreBackend[] { - return storeBackendNames().filter( - (backend): backend is ConfigStoreBackend => !STORE_BACKENDS[backend].retired, - ); -} - -export function retiredBackendNames(): RetiredStoreBackend[] { - return storeBackendNames().filter( - (backend): backend is RetiredStoreBackend => STORE_BACKENDS[backend].retired, - ); -} - -export function configBackendList(separator = ', '): string { - return configBackendNames().join(separator); -} - -export function wizardBackendChoices(): WizardStoreBackend[] { - return storeBackendNames().filter( - (backend): backend is WizardStoreBackend => - !STORE_BACKENDS[backend].retired && STORE_BACKENDS[backend].wizard, - ); -} - -export function storeFlagBackendNames(): StoreFlagBackend[] { - return storeBackendNames().filter( - (backend): backend is StoreFlagBackend => - !STORE_BACKENDS[backend].retired && STORE_BACKENDS[backend].storeFlag, - ); -} - -export function storeFlagBackendList(separator = ', '): string { - return storeFlagBackendNames().join(separator); -} - -export function isKnownStoreBackend( - backend: string | undefined | null, -): backend is StoreBackend { - return backend != null && Object.prototype.hasOwnProperty.call(STORE_BACKENDS, backend); -} - -export function getStoreBackendPolicy( - backend: string | undefined | null, -): StoreBackendPolicy | undefined { - return isKnownStoreBackend(backend) ? STORE_BACKENDS[backend] : undefined; -} - -export function isConfigStoreBackend( - backend: string | undefined | null, -): backend is ConfigStoreBackend { - return isKnownStoreBackend(backend) && !STORE_BACKENDS[backend].retired; -} - -export function isStoreFlagBackend( - backend: string | undefined | null, -): backend is StoreFlagBackend { - return isKnownStoreBackend(backend) - && !STORE_BACKENDS[backend].retired - && STORE_BACKENDS[backend].storeFlag; -} - -export function isRetiredStoreBackend( - backend: string | undefined | null, -): backend is RetiredStoreBackend { - return isKnownStoreBackend(backend) && STORE_BACKENDS[backend].retired; -} - -export function isManagedLocalBackend( - backend: string | undefined | null, -): backend is ManagedLocalStoreBackend { - return isKnownStoreBackend(backend) && STORE_BACKENDS[backend].kind === 'managed-local'; -} - -export function isExternalStoreBackend( - backend: string | undefined | null, -): backend is ExternalStoreBackend { - return isExternalBackend(backend); -} - -/** Cross from daemon runtime config into the storage factory's adapter type. */ -export function requireStorageAdapterBackend(backend: string): StorageAdapterBackend { - if (!isStorageAdapterBackend(backend)) { - throw new Error( - `Daemon runtime store backend "${backend}" is not a constructible storage adapter`, - ); - } - return backend; -} - -export { isStorageAdapterBackend }; -export type { StorageAdapterBackend }; diff --git a/packages/cli/src/store-wizard.ts b/packages/cli/src/store-wizard.ts index 768294bc71..4eac50b144 100644 --- a/packages/cli/src/store-wizard.ts +++ b/packages/cli/src/store-wizard.ts @@ -26,21 +26,6 @@ import { type ProvisionBlazegraphDockerOptions, type ProvisionBlazegraphDockerResult, } from './daemon/blazegraph-docker.js'; -import { - DEFAULT_STORE_BACKEND, - MANAGED_LOCAL_STORE_BACKEND, - STORE_BACKENDS, - configBackendList, - isConfigStoreBackend, - isRetiredStoreBackend, - isStoreFlagBackend, - storeFlagBackendList, - wizardBackendChoices, - type ConfigStoreBackend, - type ExternalStoreBackend, - type StoreFlagBackend, - type WizardStoreBackend, -} from './store-backends.js'; export interface PromptStoreBackendOptions { /** `ask` callback that closes over a shared readline interface. */ @@ -82,9 +67,9 @@ export interface PromptStoreBackendOptions { export interface PromptStoreBackendResult { /** - * Persisted store block. `null` means "leave the store block omitted"; - * daemon boot treats an omitted store block as the managed `oxigraph-server` - * default. + * Persisted store block. `null` means "use the local default" (caller + * should omit the field or set it to undefined; saveConfig drops + * undefined keys). * * `managedByDkg: true` is set by the Docker provisioner branch only; * manual URLs always get `managedByDkg: false`. The chain-reset-wipe @@ -95,8 +80,15 @@ export interface PromptStoreBackendResult { storeBlock: ExternalStoreBlock | LocalStoreBlock | null; } +// An explicit embedded/local store block carried through verbatim on an +// Enter-through re-init. `null` still means "no block — runtime default", but +// when a node already pinned a local backend with custom `options` (e.g. the +// `options.path` the oxigraph-worker adapter reads for its persistence file), +// returning that block instead of `null` keeps re-init idempotent: cli init +// writes `store: storeBlock ?? undefined`, so a `null` would clear the block +// and relocate the store on the next boot. export type LocalStoreBlock = { - backend: 'oxigraph' | 'oxigraph-persistent'; + backend: 'oxigraph' | 'oxigraph-worker' | 'oxigraph-persistent'; options?: Record; }; @@ -118,7 +110,7 @@ export type ExternalStoreBlock = // endpoints at boot. Operator-set overrides (`port`/`location`/`cacheDir`) // that planManagedOxigraph reads at boot are carried through unchanged. | { - backend: typeof MANAGED_LOCAL_STORE_BACKEND; + backend: 'oxigraph-server'; options: Record; }; @@ -144,39 +136,6 @@ function externalStoreBlock( }; } -function isSupportedExistingBackend(backend: string | undefined): backend is ConfigStoreBackend { - return isConfigStoreBackend(backend); -} - -function retiredBackendError(source: string, backend: string): Error { - const choices = source === '--store' ? storeFlagBackendList() : configBackendList(); - return new Error( - `${source} "${backend}" is no longer supported. ` + - `Use one of: ${choices}.`, - ); -} - -function unknownBackendError(source: 'prompt' | '--store', backend: string): Error { - if (source === '--store') { - return new Error(`--store must be one of: ${storeFlagBackendList()} (got "${backend}")`); - } - return new Error(`Unknown store backend "${backend}". Expected one of: ${configBackendList()}.`); -} - -function parseBackendAnswer( - input: string, - defaultBackend: string, - choices: readonly WizardStoreBackend[], -): string { - return /^\d+$/.test(input) - ? (choices[parseInt(input, 10) - 1] ?? defaultBackend) - : input.toLowerCase(); -} - -function hasStorePath(store: PromptStoreBackendOptions['existingStore'] | undefined): boolean { - return typeof store?.options?.path === 'string' && store.options.path.trim().length > 0; -} - export async function promptStoreBackend( opts: PromptStoreBackendOptions, ): Promise { @@ -197,24 +156,35 @@ export async function promptStoreBackend( : undefined; // `oxigraph-server` (daemon-managed local RocksDB server) is the default - // local backend: it gives MVCC concurrent reads + incremental persistence. - // The old `oxigraph-worker` fallback is retired; configs that still name it - // are not preserved on Enter-through. - if (isRetiredStoreBackend(existingBackend)) { - log(` Existing store.backend "${existingBackend}" is retired; defaulting to oxigraph-server.`); - } - const flagBackend = opts.flagBackend?.trim().toLowerCase(); - if (flagBackend && !isStoreFlagBackend(flagBackend)) { - if (isRetiredStoreBackend(flagBackend)) { - throw retiredBackendError('--store', flagBackend); - } - throw unknownBackendError('--store', flagBackend); - } - const defaultBackend = flagBackend - ?? (isSupportedExistingBackend(existingBackend) + // local backend: it gives MVCC concurrent reads + incremental persistence, + // whereas `oxigraph` (the embedded in-process worker) rewrites the whole + // N-Quads dump on every flush. The in-process worker stays available as a + // minimal-footprint / single-reader option. NOTE: this only changes what a + // *fresh / block-less* `dkg init` writes — the runtime fallback for configs + // with no `store` block stays `oxigraph-worker`, so the existing fleet keeps + // booting unchanged on auto-update (only an explicit re-init flips a node, + // and the daemon's STORE-SWITCH guard makes that an opt-in, not silent). + // Keep ANY explicit existing backend as the default answer — including the + // embedded `oxigraph` / `oxigraph-worker` / `oxigraph-persistent` variants. + // Keeping the *exact* variant (rather than normalising the worker variants + // onto the listed `oxigraph` choice) matters for distinguishing intent: an + // Enter-through resolves the default back to that exact backend (so the + // preserve branch below keeps the block + custom options), whereas explicitly + // picking option `2` ("oxigraph") resolves to a *different* answer and is + // treated as a real switch to the plain embedded worker. Only a truly absent + // store config (fresh install / block-less node) falls through to the new + // `oxigraph-server` default. + const defaultBackend = opts.flagBackend + ?? (existingBackend === 'blazegraph' || existingBackend === 'sparql-http' || existingBackend === 'oxigraph-server' + || existingBackend === 'oxigraph' || existingBackend === 'oxigraph-worker' || existingBackend === 'oxigraph-persistent' ? existingBackend - : DEFAULT_STORE_BACKEND); - const backendChoices = wizardBackendChoices(); + : 'oxigraph-server'); + const backendChoices = ['oxigraph-server', 'oxigraph', 'blazegraph'] as const; + const backendLabels: Record = { + 'oxigraph-server': 'oxigraph-server (managed local server — recommended)', + 'oxigraph': 'oxigraph (embedded in-process worker)', + 'blazegraph': 'blazegraph (external SPARQL endpoint)', + }; // `sparql-http` is intentionally not listed (advanced bring-your-own-server // option) but is still accepted when typed or inherited from an existing // config / `--store` flag. Resolve the default *answer* by name for unlisted @@ -228,7 +198,7 @@ export async function promptStoreBackend( log(' Triple store backend:'); for (let i = 0; i < backendChoices.length; i++) { const choice = backendChoices[i]; - log(` ${i + 1}) ${STORE_BACKENDS[choice].label}`); + log(` ${i + 1}) ${backendLabels[choice] ?? choice}`); } // When the inherited/flagged backend isn't one of the numbered choices // (e.g. `sparql-http`), spell out that pressing Enter keeps it and that a @@ -246,52 +216,49 @@ export async function promptStoreBackend( // out-of-range number (typo like "4") falls back to `defaultBackend` — // i.e. the recommended option shown to the operator — rather than a // hard-coded `oxigraph`, so a fat-fingered digit on a fresh install no - // longer silently downgrades the node to an embedded backend. - const backendAnswer = parseBackendAnswer(backendInput, defaultBackend, backendChoices); - - if (isRetiredStoreBackend(backendAnswer)) { - throw retiredBackendError('store backend', backendAnswer); - } - if (!isSupportedExistingBackend(backendAnswer)) { - throw unknownBackendError('prompt', backendAnswer); - } - const backendPolicy = STORE_BACKENDS[backendAnswer]; + // longer silently downgrades the node to the embedded worker. + const backendAnswer = /^\d+$/.test(backendInput) + ? (backendChoices[parseInt(backendInput, 10) - 1] ?? defaultBackend) + : backendInput.toLowerCase(); // `oxigraph-server` (daemon-managed local server) is the default numbered // choice and is also accepted by name. No URL prompt or probe: the endpoint // doesn't exist until the daemon spawns it at boot. - if (backendPolicy.kind === 'managed-local') { + if (backendAnswer === 'oxigraph-server') { log(' Using a daemon-managed local Oxigraph server (started on first daemon boot).'); // Preserve existing managed-server overrides (port/location/cacheDir) on an // Enter-through: `dkg init` persists this block, so returning empty options // would silently reset a custom port/RocksDB path on the next boot — the // same hazard applyStoreFlagsToConfig guards against on the `--store` path. const prevOptions = - existingBackend === MANAGED_LOCAL_STORE_BACKEND && opts.existingStore?.options + existingBackend === 'oxigraph-server' && opts.existingStore?.options ? opts.existingStore.options : {}; - return { storeBlock: { backend: MANAGED_LOCAL_STORE_BACKEND, options: prevOptions } }; + return { storeBlock: { backend: 'oxigraph-server', options: prevOptions } }; } - if (backendPolicy.kind === 'local') { - const localBackend = backendAnswer as LocalStoreBlock['backend']; - if (STORE_BACKENDS[localBackend].requiresExistingPath && !hasStorePath(opts.existingStore)) { - throw new Error( - 'store backend "oxigraph-persistent" requires store.options.path; ' + - 'set it manually in config.json or use "oxigraph-server".', - ); + if (backendAnswer !== 'blazegraph' && backendAnswer !== 'sparql-http') { + // Embedded in-process worker. Preserve an existing explicit local store + // block verbatim ONLY when the operator kept that same backend — i.e. the + // resolved answer equals the existing backend (an Enter-through, which + // resolves the default back to the exact existing variant). That keeps a + // re-init idempotent and never drops custom `options` (e.g. the worker's + // `options.path`) or relocates the store. An *explicit* switch — picking + // option `2`/"oxigraph" on a node currently using `oxigraph-worker` / + // `oxigraph-persistent`, a switch from an external/server backend, or a + // fresh / block-less init — resolves to a different answer and falls + // through to `null`, so `dkg init` clears the old block as intended. + if ( + (backendAnswer === 'oxigraph' || + backendAnswer === 'oxigraph-worker' || + backendAnswer === 'oxigraph-persistent') && + backendAnswer === existingBackend + ) { + return { storeBlock: { backend: existingBackend, options: opts.existingStore?.options } }; } - const prevOptions = - localBackend === existingBackend && opts.existingStore?.options - ? opts.existingStore.options - : {}; - return { storeBlock: { backend: localBackend, options: prevOptions } }; - } - - if (backendPolicy.kind !== 'external') { - throw unknownBackendError('prompt', backendAnswer); + return { storeBlock: null }; } - const backend = backendAnswer as ExternalStoreBackend; + const backend = backendAnswer as 'blazegraph' | 'sparql-http'; // URL prompt loop: validate each attempt, surface the operator-facing // failure message, allow retry or abort. @@ -350,13 +317,14 @@ export async function promptStoreBackend( } const retry = (await opts.ask('Retry with a URL? (y/n)', 'y')).toLowerCase(); if (retry === 'n') { - log(' Aborting store setup; using the oxigraph-server default.'); + log(' Aborting store setup; defaulting to local Oxigraph.'); return { storeBlock: null }; } continue; } - const optionsForProbe = { [backendPolicy.queryEndpointOption]: url }; + const optionsForProbe = + backend === 'blazegraph' ? { url } : { queryEndpoint: url }; const health = await checkExternalStoreReachable({ storeConfig: { backend, options: optionsForProbe }, fetch: opts.fetch, @@ -384,7 +352,7 @@ export async function promptStoreBackend( 'y', )).toLowerCase(); if (retry === 'n') { - log(' Aborting store setup; using the oxigraph-server default.'); + log(' Aborting store setup; defaulting to local Oxigraph.'); return { storeBlock: null }; } } @@ -423,64 +391,61 @@ export async function applyStoreFlagsToConfig( opts: ApplyStoreFlagsOptions, ): Promise { const log = opts.log ?? console.log; - const backend = opts.storeFlag?.trim().toLowerCase(); + const backend = opts.storeFlag; if (!backend) return; const load = opts.loadConfig ?? loadConfig; const save = opts.saveConfig ?? saveConfig; - if (isRetiredStoreBackend(backend)) { - throw retiredBackendError('--store', backend); - } - if (!isStoreFlagBackend(backend)) { - throw unknownBackendError('--store', backend); - } - const backendPolicy = STORE_BACKENDS[backend]; - - // Operators who pass `--store oxigraph` are explicitly opting into the - // embedded development store. Persist it because an omitted store block now - // means the managed `oxigraph-server` default. - if (backendPolicy.kind === 'local') { - const localBackend = backend as Extract; + // Operators who pass `--store oxigraph` may be trying to FORCE local + // even though their existing config has a `store` block — honour + // that by clearing the block. + if ( + backend === 'oxigraph' || + backend === 'oxigraph-worker' || + backend === 'oxigraph-persistent' + ) { const existing = await load(); - await save({ ...existing, store: { backend: localBackend, options: {} } }); - if (localBackend === 'oxigraph') { - log(' Store configured: oxigraph (embedded development store).'); - } else { - log(` Store configured: ${localBackend}.`); + if (existing.store) { + log(` Removing existing store block (--store ${backend} → local default).`); + const next = { ...existing }; + delete next.store; + await save(next); } return; } // Daemon-managed local Oxigraph server: no URL to validate (the daemon // brings it up at boot). Write the block and return. - if (backendPolicy.kind === 'managed-local') { + if (backend === 'oxigraph-server') { const existing = await load(); // Preserve any existing managed-server overrides (port/location/cacheDir) // that planManagedOxigraph reads at boot — re-running setup with // `--store oxigraph-server` must not silently reset them to defaults. const prevOptions = - existing.store?.backend === MANAGED_LOCAL_STORE_BACKEND && existing.store.options + existing.store?.backend === 'oxigraph-server' && existing.store.options ? existing.store.options : {}; - await save({ ...existing, store: { backend: MANAGED_LOCAL_STORE_BACKEND, options: prevOptions } }); + await save({ ...existing, store: { backend: 'oxigraph-server', options: prevOptions } }); log(' Store configured: oxigraph-server (daemon-managed local server).'); return; } - if (backendPolicy.kind !== 'external') { - throw unknownBackendError('--store', backend); + if (backend !== 'blazegraph' && backend !== 'sparql-http') { + throw new Error( + `--store must be one of: oxigraph, blazegraph, sparql-http, oxigraph-server (got "${backend}")`, + ); } - const externalBackend = backend as ExternalStoreBackend; const url = opts.storeUrlFlag?.trim(); if (!url) { - throw new Error(`--store ${externalBackend} requires --store-url `); + throw new Error(`--store ${backend} requires --store-url `); } - const optionsForProbe = { [backendPolicy.queryEndpointOption]: url }; + const optionsForProbe = + backend === 'blazegraph' ? { url } : { queryEndpoint: url }; const health = await checkExternalStoreReachable({ - storeConfig: { backend: externalBackend, options: optionsForProbe }, + storeConfig: { backend, options: optionsForProbe }, fetch: opts.fetch, }); if (!health.ok) { @@ -490,8 +455,8 @@ export async function applyStoreFlagsToConfig( const existing = await load(); const next: DkgConfig = { ...existing, - store: externalStoreBlock(externalBackend, url, false), + store: externalStoreBlock(backend, url, false), }; await save(next); - log(` Store configured: ${externalBackend} (${url}) — verified reachable.`); + log(` Store configured: ${backend} (${url}) — verified reachable.`); } diff --git a/packages/cli/test/chain-reset-wipe.test.ts b/packages/cli/test/chain-reset-wipe.test.ts index 6db6e56d4d..909ebce67b 100644 --- a/packages/cli/test/chain-reset-wipe.test.ts +++ b/packages/cli/test/chain-reset-wipe.test.ts @@ -589,7 +589,7 @@ describe('chainResetWipe — external SPARQL wipe', () => { expect(result.failedFiles.some((f) => f.error.includes('ECONNREFUSED'))).toBe(true); }); - it('skips external wipe entirely for local backends (storeConfig present, backend oxigraph-persistent)', async () => { + it('skips external wipe entirely for local backends (storeConfig present, backend oxigraph-worker)', async () => { writeFileSync(join(dataDir, 'store.nq'), '

.'); const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); @@ -597,7 +597,7 @@ describe('chainResetWipe — external SPARQL wipe', () => { dataDir, currentMarker: NEW_MARKER, storeConfig: { - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', // Options that LOOK like an external URL: these must be ignored // because the backend itself is local. Otherwise an operator who // hand-tuned options would see surprise SPARQL requests. @@ -652,7 +652,7 @@ describe('detectBackendSwitch', () => { const logs: string[] = []; const result = detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-persistent', + currentBackend: 'oxigraph-worker', acceptStoreReset: false, log: (m) => logs.push(m), }); @@ -662,7 +662,7 @@ describe('detectBackendSwitch', () => { expect(result.aborted).toBe(false); const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); - expect(persisted.lastBackend).toBe('oxigraph-persistent'); + expect(persisted.lastBackend).toBe('oxigraph-worker'); // First-boot path is silent — no STORE-SWITCH warning header. expect(logs.find((l) => l.includes('STORE-SWITCH'))).toBeUndefined(); }); @@ -696,19 +696,19 @@ describe('detectBackendSwitch', () => { it('is a no-op when backend matches previous boot', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), ); const logs: string[] = []; const result = detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-persistent', + currentBackend: 'oxigraph-worker', acceptStoreReset: false, log: (m) => logs.push(m), }); expect(result.changed).toBe(false); - expect(result.previous).toBe('oxigraph-persistent'); + expect(result.previous).toBe('oxigraph-worker'); expect(result.aborted).toBe(false); expect(logs).toEqual([]); }); @@ -716,7 +716,7 @@ describe('detectBackendSwitch', () => { it('aborts boot on mismatch without acceptStoreReset, surfaces multi-line warning', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), ); const logs: string[] = []; @@ -728,25 +728,25 @@ describe('detectBackendSwitch', () => { }); expect(result.changed).toBe(true); - expect(result.previous).toBe('oxigraph-persistent'); + expect(result.previous).toBe('oxigraph-worker'); expect(result.aborted).toBe(true); const joined = logs.join('\n'); expect(joined).toMatch(/STORE-SWITCH/); - expect(joined).toMatch(/previous: oxigraph-persistent/); + expect(joined).toMatch(/previous: oxigraph-worker/); expect(joined).toMatch(/current:\s+blazegraph/); expect(joined).toMatch(/DKG_ACCEPT_STORE_RESET=1/); // State file MUST still record the old backend so a corrected // config (operator reverts the edit) sees a match on next boot. const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); - expect(persisted.lastBackend).toBe('oxigraph-persistent'); + expect(persisted.lastBackend).toBe('oxigraph-worker'); }); it('proceeds and updates state when acceptStoreReset=true', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), ); const logs: string[] = []; @@ -775,12 +775,12 @@ describe('detectBackendSwitch', () => { // Establish a baseline by running detectBackendSwitch first. detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-persistent', + currentBackend: 'oxigraph-worker', acceptStoreReset: false, }); expect( JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')).lastBackend, - ).toBe('oxigraph-persistent'); + ).toBe('oxigraph-worker'); // Now run a marker change — the wipe path persists chainResetMarker // and MUST preserve lastBackend. @@ -792,7 +792,7 @@ describe('detectBackendSwitch', () => { const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); expect(persisted.chainResetMarker).toBe(NEW_MARKER); - expect(persisted.lastBackend).toBe('oxigraph-persistent'); + expect(persisted.lastBackend).toBe('oxigraph-worker'); }); }); diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index 3f2e3a10b4..c4f6a30152 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -102,7 +102,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: authEnabled }, store: { - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', options: { path: join(home, 'store.nq') }, }, // Real EVM adapter against the shared Hardhat node (port 9548 per diff --git a/packages/cli/test/daemon-http-inflight-cap.test.ts b/packages/cli/test/daemon-http-inflight-cap.test.ts index c36f9b0749..e38b58c48f 100644 --- a/packages/cli/test/daemon-http-inflight-cap.test.ts +++ b/packages/cli/test/daemon-http-inflight-cap.test.ts @@ -1,5 +1,4 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { request as httpRequest } from 'node:http'; import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from './helpers/live-daemon.js'; /** @@ -12,11 +11,13 @@ import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from '. * http-admission-control.test.ts: it would fail if the limiter were never wired * into createServer, wired after an early return, or never released. * - * Saturation is created with a real HTTP request whose JSON body is deliberately - * left unfinished. The test then polls the admission-exempt status route until - * it observes the occupied slot before sending competing requests. This proves - * the production wiring without relying on a fast request burst happening to - * overlap on a particular runner. + * Saturation is created with a 50-request burst against cap=1 rather than a + * single held-open request. That is statistically deterministic — 50 concurrent + * requests cannot all serialize through one slot without overlap — and avoids a + * brittle blocking fixture (the daemon admits before the route reads the body, + * so an unfinished-body "hold" does not reliably pin the slot). The precise + * one-in/one-shed/release semantics are covered deterministically by the unit + * tests; here we prove the wiring end-to-end. */ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { let daemon: LiveDaemon | undefined; @@ -45,76 +46,24 @@ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { }); } - async function readAdmission( - d: LiveDaemon, - ): Promise<{ inFlight: number; max: number; rejectedTotal: number }> { - const res = await fetch(`${d.base}/api/status`, { headers: authHeaders(d) }); - expect(res.status).toBe(200); - const body = (await res.json()) as { - admission?: { inFlight: number; max: number; rejectedTotal: number }; - }; - expect(body.admission).toBeDefined(); - return body.admission!; - } - - async function holdQuerySlot(d: LiveDaemon): Promise<{ - release: () => void; - responseStatus: Promise; - }> { - let released = false; - let req: ReturnType; - const responseStatus = new Promise((resolve, reject) => { - req = httpRequest( - `${d.base}/api/query`, - { - method: 'POST', - headers: authHeaders(d), - }, - (res) => { - res.resume(); - res.once('end', () => resolve(res.statusCode ?? 0)); - }, - ); - req.once('error', reject); - req.write('{"sparql":"SELECT * WHERE { ?s ?p ?o } LIMIT 1","hold":"'); - }); - const release = () => { - if (released) return; - released = true; - req.end('released"}'); - }; - - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - if ((await readAdmission(d)).inFlight === 1) { - return { release, responseStatus }; - } - await new Promise((resolve) => setTimeout(resolve, 20)); - } - release(); - throw new Error('Timed out waiting for the held query to occupy the admission slot'); - } - it('sheds concurrent over-capacity requests with 503 + Retry-After, then recovers', async () => { const d = daemon!; - const held = await holdQuerySlot(d); - try { - const results = await Promise.all( - Array.from({ length: 10 }, () => - selectQuery(d) - .then((r) => ({ status: r.status, retryAfter: r.headers.get('retry-after') })) - .catch(() => ({ status: 0, retryAfter: null as string | null })), - ), - ); - - expect(results.every((r) => r.status === 503)).toBe(true); - expect(results.every((r) => r.retryAfter === '1')).toBe(true); - - held.release(); - expect(await held.responseStatus).toBe(200); - } finally { - held.release(); - } + const results = await Promise.all( + Array.from({ length: 50 }, () => + selectQuery(d) + .then((r) => ({ status: r.status, retryAfter: r.headers.get('retry-after') })) + .catch(() => ({ status: 0, retryAfter: null as string | null })), + ), + ); + const shed = results.filter((r) => r.status === 503); + const ok = results.filter((r) => r.status === 200); + + // Every result must be an EXPECTED status — never a network error (0) or an + // unexpected 4xx/5xx that would otherwise hide behind the >=1/>=1 counts. + expect(results.every((r) => r.status === 200 || r.status === 503)).toBe(true); + expect(ok.length).toBeGreaterThan(0); // at least one admitted + expect(shed.length).toBeGreaterThan(0); // cap enforced under concurrent load + expect(shed.every((r) => r.retryAfter === '1')).toBe(true); // Retry-After present on every 503 // Slots are released after each handler completes → a fresh request succeeds. const recovered = await selectQuery(d); @@ -123,54 +72,58 @@ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { it('keeps the exempt liveness path (/api/status) answerable even while saturated', async () => { const d = daemon!; - const held = await holdQuerySlot(d); - try { - const [statuses, burstStatuses] = await Promise.all([ - Promise.all( - Array.from({ length: 12 }, () => - fetch(`${d.base}/api/status`, { headers: authHeaders(d) }) - .then((r) => r.status) - .catch(() => 0), - ), - ), - Promise.all( - Array.from({ length: 10 }, () => - selectQuery(d).then((r) => r.status).catch(() => 0), - ), - ), - ]); - - expect(statuses.every((s) => s === 200)).toBe(true); - expect(burstStatuses.every((s) => s === 503)).toBe(true); - } finally { - held.release(); - } - expect(await held.responseStatus).toBe(200); + // Saturate with non-exempt query work; capture the burst results so we can + // PROVE the daemon was actually over capacity (>=1 shed) while the status + // probes ran — otherwise "status stayed 200" would be vacuous. + const burst = Promise.all( + Array.from({ length: 40 }, () => + selectQuery(d).then((r) => r.status).catch(() => 0), + ), + ); + // ...while hammering the exempt status endpoint, which must always answer 200. + const statuses = await Promise.all( + Array.from({ length: 12 }, () => + fetch(`${d.base}/api/status`, { headers: authHeaders(d) }) + .then((r) => r.status) + .catch(() => 0), + ), + ); + const burstStatuses = await burst; + + expect(statuses.every((s) => s === 200)).toBe(true); // exempt path never shed + expect(burstStatuses.filter((s) => s === 503).length).toBeGreaterThan(0); // saturation really happened + expect(burstStatuses.every((s) => s === 200 || s === 503)).toBe(true); // no unexpected failures }, 60_000); it('surfaces admission stats on /api/status (effective cap + per-burst shed delta)', async () => { const d = daemon!; + // Read the surfaced admission block off the exempt status endpoint. + const readAdmission = async (): Promise<{ inFlight: number; max: number; rejectedTotal: number }> => { + const res = await fetch(`${d.base}/api/status`, { headers: authHeaders(d) }); + expect(res.status).toBe(200); + const body = (await res.json()) as { + admission?: { inFlight: number; max: number; rejectedTotal: number }; + }; + expect(body.admission).toBeDefined(); + return body.admission!; + }; + // Snapshot BEFORE this burst — earlier tests in this file already shed, so a // bare `rejectedTotal > 0` would pass without proving THIS burst moved the // counter (i.e. that the surfaced value still tracks live shedding). - const before = await readAdmission(d); + const before = await readAdmission(); expect(before.max).toBe(1); // the pinned effective cap is surfaced expect(typeof before.inFlight).toBe('number'); - const held = await holdQuerySlot(d); - try { - const burst = await Promise.all( - Array.from({ length: 10 }, () => selectQuery(d).then((r) => r.status).catch(() => 0)), - ); - expect(burst.every((s) => s === 503)).toBe(true); - } finally { - held.release(); - } - expect(await held.responseStatus).toBe(200); + // Saturate the non-exempt path so this burst provably sheds. + const burst = await Promise.all( + Array.from({ length: 50 }, () => selectQuery(d).then((r) => r.status).catch(() => 0)), + ); + expect(burst.filter((s) => s === 503).length).toBeGreaterThan(0); // this burst really shed // /api/status is admission-exempt, so reading it doesn't perturb the counter: // `after` MUST exceed `before` by the sheds we just caused. - const after = await readAdmission(d); + const after = await readAdmission(); expect(after.rejectedTotal).toBeGreaterThan(before.rejectedTotal); }, 60_000); }); diff --git a/packages/cli/test/daemon-startup-validation.test.ts b/packages/cli/test/daemon-startup-validation.test.ts index cee7688fb6..cef225b46b 100644 --- a/packages/cli/test/daemon-startup-validation.test.ts +++ b/packages/cli/test/daemon-startup-validation.test.ts @@ -1,5 +1,5 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { computeNetworkId } from '../../core/src/genesis.js'; @@ -9,9 +9,6 @@ const mocks = vi.hoisted(() => ({ agentCreate: vi.fn(), loadOpWallets: vi.fn(), loadNetworkConfig: vi.fn(), - checkExternalStoreReachable: vi.fn(), - checkOrSetStoreIdentity: vi.fn(), - startManagedOxigraph: vi.fn(), })); vi.mock('@origintrail-official/dkg-agent', async importOriginal => { @@ -32,23 +29,6 @@ vi.mock('../src/config.js', async importOriginal => { }; }); -vi.mock('../src/daemon/oxigraph-managed.js', async importOriginal => { - const actual = await importOriginal(); - return { - ...actual, - startManagedOxigraph: mocks.startManagedOxigraph, - }; -}); - -vi.mock('../src/daemon/store-health-check.js', async importOriginal => { - const actual = await importOriginal(); - return { - ...actual, - checkExternalStoreReachable: mocks.checkExternalStoreReachable, - checkOrSetStoreIdentity: mocks.checkOrSetStoreIdentity, - }; -}); - const { runDaemonInner } = await import('../src/daemon/lifecycle.js'); function closeDashboardDbFromAgentCreateArg(createArg: any): void { @@ -58,29 +38,6 @@ function closeDashboardDbFromAgentCreateArg(createArg: any): void { db?.close?.(); } -function managedOxigraphResult(dataDir: string) { - return { - handle: { - queryEndpoint: 'http://127.0.0.1:12001/query', - updateEndpoint: 'http://127.0.0.1:12001/update', - killSync: vi.fn(), - }, - storeConfig: { - backend: 'sparql-http', - options: { - queryEndpoint: 'http://127.0.0.1:12001/query', - updateEndpoint: 'http://127.0.0.1:12001/update', - managedByDkg: true, - }, - }, - largeLiteralStorage: { enabled: true, directory: join(dataDir, 'literal-blobs') }, - sharedMemoryPublicSnapshotStorage: { - enabled: true, - directory: join(dataDir, 'swm-public-snapshots'), - }, - }; -} - describe('daemon startup network validation', () => { let tempHome: string | undefined; let originalDkgHome: string | undefined; @@ -88,15 +45,6 @@ describe('daemon startup network validation', () => { let stderrWrite: typeof process.stderr.write = process.stderr.write; let uncaughtExceptionListeners: NodeJS.UncaughtExceptionListener[] = []; let unhandledRejectionListeners: NodeJS.UnhandledRejectionListener[] = []; - const originalAcceptStoreReset = process.env.DKG_ACCEPT_STORE_RESET; - - beforeEach(() => { - mocks.loadNetworkConfig.mockResolvedValue(undefined); - mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); - mocks.startManagedOxigraph.mockResolvedValue(null); - mocks.checkExternalStoreReachable.mockResolvedValue({ ok: true, backend: 'sparql-http', endpoint: 'http://127.0.0.1:12001/query' }); - mocks.checkOrSetStoreIdentity.mockResolvedValue({ ok: true, action: 'matched', nodeName: 'test-node' }); - }); afterEach(async () => { vi.restoreAllMocks(); @@ -116,142 +64,10 @@ describe('daemon startup network validation', () => { } else { process.env.DKG_HOME = originalDkgHome; } - if (originalAcceptStoreReset === undefined) { - delete process.env.DKG_ACCEPT_STORE_RESET; - } else { - process.env.DKG_ACCEPT_STORE_RESET = originalAcceptStoreReset; - } if (tempHome) await rm(tempHome, { recursive: true, force: true }); tempHome = undefined; }); - async function useTempHome(prefix: string) { - tempHome = await mkdtemp(join(tmpdir(), prefix)); - originalDkgHome = process.env.DKG_HOME; - process.env.DKG_HOME = tempHome; - stdoutWrite = process.stdout.write; - stderrWrite = process.stderr.write; - uncaughtExceptionListeners = process.listeners('uncaughtException') as NodeJS.UncaughtExceptionListener[]; - unhandledRejectionListeners = process.listeners('unhandledRejection') as NodeJS.UnhandledRejectionListener[]; - return vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - } - - it('exits before managed store startup when a blockless config has legacy store.nq and no reset acknowledgement', async () => { - const stdoutSpy = await useTempHome('dkg-legacy-store-gate-'); - await writeFile(join(tempHome!, 'store.nq'), '

.'); - vi - .spyOn(process, 'exit') - .mockImplementation(((code?: string | number | null) => { - throw new Error(`process.exit:${code}`); - }) as never); - - await expect(runDaemonInner(true, { - name: 'legacy-store-gate-test', - listenPort: 0, - nodeRole: 'edge', - } as any, Date.now())).rejects.toThrow('process.exit:1'); - - const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); - expect(output).toContain('legacy store.nq from the old implicit worker default'); - expect(output).toContain('DKG_ACCEPT_STORE_RESET=1'); - expect(mocks.startManagedOxigraph).not.toHaveBeenCalled(); - expect(mocks.agentCreate).not.toHaveBeenCalled(); - }); - - it('continues with the effective oxigraph-server store after legacy store.nq is acknowledged', async () => { - const stdoutSpy = await useTempHome('dkg-legacy-store-ack-'); - process.env.DKG_ACCEPT_STORE_RESET = '1'; - await writeFile(join(tempHome!, 'store.nq'), '

.'); - mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome!)); - mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); - - await expect(runDaemonInner(true, { - name: 'legacy-store-ack-test', - listenPort: 0, - nodeRole: 'edge', - } as any, Date.now())).rejects.toThrow('after-agent-create'); - - const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); - expect(output).toContain('using oxigraph-server'); - expect(mocks.startManagedOxigraph).toHaveBeenCalledTimes(1); - expect(mocks.startManagedOxigraph.mock.calls[0]?.[0]).toMatchObject({ - dataDir: tempHome, - config: { - store: { backend: 'oxigraph-server', options: {} }, - }, - }); - expect(mocks.agentCreate).toHaveBeenCalledTimes(1); - expect(mocks.agentCreate.mock.calls[0]?.[0]).toMatchObject({ - storeConfig: { - backend: 'sparql-http', - options: { - queryEndpoint: 'http://127.0.0.1:12001/query', - updateEndpoint: 'http://127.0.0.1:12001/update', - managedByDkg: true, - }, - }, - largeLiteralStorage: { enabled: true, directory: join(tempHome!, 'literal-blobs') }, - sharedMemoryPublicSnapshotStorage: { enabled: true, directory: join(tempHome!, 'swm-public-snapshots') }, - }); - }); - - it('blocks a wizard-rewritten oxigraph-server config with legacy store.nq and no backend marker', async () => { - const stdoutSpy = await useTempHome('dkg-rewritten-legacy-store-gate-'); - await writeFile(join(tempHome!, 'store.nq'), '

.'); - vi - .spyOn(process, 'exit') - .mockImplementation(((code?: string | number | null) => { - throw new Error(`process.exit:${code}`); - }) as never); - - await expect(runDaemonInner(true, { - name: 'rewritten-legacy-store-gate-test', - listenPort: 0, - nodeRole: 'edge', - store: { backend: 'oxigraph-server', options: {} }, - } as any, Date.now())).rejects.toThrow('process.exit:1'); - - const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); - expect(output).toContain('legacy store.nq from the old worker-backed store'); - expect(output).toContain('DKG_ACCEPT_STORE_RESET=1'); - expect(mocks.startManagedOxigraph).not.toHaveBeenCalled(); - expect(mocks.agentCreate).not.toHaveBeenCalled(); - }); - - it('migrates an explicit legacy worker config after reset acknowledgement', async () => { - await useTempHome('dkg-explicit-legacy-store-ack-'); - process.env.DKG_ACCEPT_STORE_RESET = '1'; - await writeFile(join(tempHome!, 'store.nq'), '

.'); - mocks.startManagedOxigraph.mockResolvedValue({ - handle: { queryEndpoint: 'http://127.0.0.1:12001/query', updateEndpoint: 'http://127.0.0.1:12001/update', killSync: vi.fn() }, - storeConfig: { - backend: 'sparql-http', - options: { - queryEndpoint: 'http://127.0.0.1:12001/query', - updateEndpoint: 'http://127.0.0.1:12001/update', - managedByDkg: true, - }, - }, - largeLiteralStorage: { enabled: true, directory: join(tempHome!, 'literal-blobs') }, - sharedMemoryPublicSnapshotStorage: { enabled: true, directory: join(tempHome!, 'swm-public-snapshots') }, - }); - mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); - - await expect(runDaemonInner(true, { - name: 'explicit-legacy-store-ack-test', - listenPort: 0, - nodeRole: 'edge', - store: { backend: 'oxigraph-worker' }, - } as any, Date.now())).rejects.toThrow('after-agent-create'); - - expect(mocks.startManagedOxigraph.mock.calls[0]?.[0]).toMatchObject({ - config: { store: { backend: 'oxigraph-server', options: {} } }, - }); - expect(mocks.agentCreate.mock.calls[0]?.[0]).toMatchObject({ - storeConfig: { backend: 'sparql-http' }, - }); - }); - it('exits before agent creation when the selected network is pre-deployment', async () => { tempHome = await mkdtemp(join(tmpdir(), 'dkg-predeployment-startup-')); originalDkgHome = process.env.DKG_HOME; @@ -344,7 +160,6 @@ describe('daemon startup network validation', () => { defaultNodeRole: 'edge', }); mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); - mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome)); mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); @@ -399,7 +214,6 @@ describe('daemon startup network validation', () => { defaultNodeRole: 'edge', }); mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); - mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome)); mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); diff --git a/packages/cli/test/daemon-state.test.ts b/packages/cli/test/daemon-state.test.ts deleted file mode 100644 index 0c158fc8eb..0000000000 --- a/packages/cli/test/daemon-state.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { mkdtemp, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { - DAEMON_STATE_FILE, - readPersistedDaemonState, - readPersistedNetworkConfig, - readPersistedStoreBackend, - writePersistedChainResetMarker, - writePersistedNetworkConfig, - writePersistedStoreBackend, -} from '../src/daemon/daemon-state.js'; - -describe('persisted daemon state', () => { - it('preserves sibling fields when marker, backend, and network writers update independently', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-daemon-state-')); - - writePersistedStoreBackend(dataDir, 'oxigraph-server'); - writePersistedNetworkConfig(dataDir, 'mainnet-gnosis'); - writePersistedChainResetMarker(dataDir, 'reset-42'); - - expect(readPersistedDaemonState(dataDir)).toMatchObject({ - chainResetMarker: 'reset-42', - lastBackend: 'oxigraph-server', - lastNetworkConfig: 'mainnet-gnosis', - }); - expect(readPersistedStoreBackend(dataDir)).toBe('oxigraph-server'); - expect(readPersistedNetworkConfig(dataDir)).toBe('mainnet-gnosis'); - - writePersistedStoreBackend(dataDir, 'blazegraph'); - expect(readPersistedDaemonState(dataDir)).toMatchObject({ - chainResetMarker: 'reset-42', - lastBackend: 'blazegraph', - lastNetworkConfig: 'mainnet-gnosis', - }); - }); - - it('treats malformed state as absent and repairs it on the next write', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-daemon-state-invalid-')); - await writeFile( - join(dataDir, DAEMON_STATE_FILE), - JSON.stringify({ chainResetMarker: 42, lastBackend: 'oxigraph' }), - ); - - expect(readPersistedDaemonState(dataDir)).toBeNull(); - writePersistedNetworkConfig(dataDir, 'testnet'); - expect(readPersistedDaemonState(dataDir)).toMatchObject({ - chainResetMarker: null, - lastNetworkConfig: 'testnet', - }); - }); -}); diff --git a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts index 5934ab3650..98840f7494 100644 --- a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts +++ b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts @@ -148,7 +148,6 @@ describe('runDaemonInner StorageACK timing wiring', () => { networkConfig: 'mainnet-gnosis', listenPort: 0, nodeRole: 'core', - store: { backend: 'oxigraph' }, chain: { type: 'evm', rpcUrl: 'https://private-rpc.example', @@ -286,7 +285,6 @@ describe('runDaemonInner StorageACK timing wiring', () => { listenPort: 0, nodeRole: 'edge', apiPort: 0, - store: { backend: 'oxigraph' }, auth: { enabled: false }, promoteQueue: { enabled: false }, publisher: { enabled: true }, diff --git a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts index 3f4eaa1636..07eb5d57b7 100644 --- a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts +++ b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts @@ -114,7 +114,6 @@ describe('runDaemonInner wires sync options into DKGAgent.create', () => { networkConfig: 'mainnet-gnosis', listenPort: 0, nodeRole: 'core', - store: { backend: 'oxigraph' }, chain: { type: 'evm', rpcUrl: 'https://private-rpc.example', diff --git a/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts b/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts index 3204585822..c1617aa888 100644 --- a/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts +++ b/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts @@ -62,7 +62,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: true }, store: { - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', options: { path: join(home, 'store.nq') }, }, chain: { diff --git a/packages/cli/test/handle-request-store-persistence.test.ts b/packages/cli/test/handle-request-store-persistence.test.ts deleted file mode 100644 index 0afd3d0305..0000000000 --- a/packages/cli/test/handle-request-store-persistence.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createServer } from 'node:http'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import type { AddressInfo } from 'node:net'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterEach, describe, expect, it, vi } from 'vitest'; -import type { DkgConfig } from '../src/config.js'; -import { handleRequest } from '../src/daemon/handle-request.js'; -import { createRequestStoreContext } from '../src/daemon/routes/context.js'; -import type { StoreRuntimeContext } from '../src/daemon/store-runtime.js'; - -describe('handleRequest store config boundary', () => { - afterEach(() => { - vi.unstubAllEnvs(); - }); - - it('persists the operator config rather than the materialized managed-store runtime', async () => { - const home = await mkdtemp(join(tmpdir(), 'dkg-handle-request-store-')); - vi.stubEnv('DKG_HOME', home); - - const operatorConfig: DkgConfig = { - name: 'operator-config-route-test', - nodeRole: 'edge', - chain: { type: 'mock' }, - }; - const storeRuntime: StoreRuntimeContext = { - operatorConfig, - effectiveStore: { backend: 'oxigraph-server', options: {} }, - runtimeStore: { - backend: 'sparql-http', - options: { - queryEndpoint: 'http://127.0.0.1:7878/query', - updateEndpoint: 'http://127.0.0.1:7878/update', - managedByDkg: true, - }, - }, - }; - const requestStoreContext = createRequestStoreContext(storeRuntime); - expect(requestStoreContext).toEqual({ - config: operatorConfig, - effectiveStore: storeRuntime.effectiveStore, - runtimeStore: storeRuntime.runtimeStore, - }); - expect(requestStoreContext).not.toHaveProperty('operatorConfig'); - expect(requestStoreContext).not.toHaveProperty('storeRuntime'); - - const server = createServer((req, res) => { - const args: Parameters = [ - req, - res, - { resolveAgentAddress: () => 'operator-config-route-test' } as any, - {} as any, - null, - storeRuntime, - Date.now(), - {} as any, - { wallets: [] }, - null, - {} as any, - {} as any, - undefined, - '0.0.0-test', - '', - {} as any, - {} as any, - new Map(), - new Map(), - {} as any, - null, - new Set(), - '127.0.0.1', - { value: 0 }, - [], - { inFlight: 0, max: 0, rejectedTotal: 0 }, - ]; - void handleRequest(...args).catch(() => { - res.statusCode = 500; - res.end('route failed'); - }); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - try { - const address = server.address() as AddressInfo; - const response = await fetch(`http://127.0.0.1:${address.port}/api/register-adapter`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ id: 'openclaw' }), - }); - expect(response.status).toBe(200); - - const persisted = JSON.parse(await readFile(join(home, 'config.json'), 'utf8')) as DkgConfig; - expect(persisted.store).toBeUndefined(); - expect(persisted.localAgentIntegrations?.openclaw?.enabled).toBe(true); - expect(JSON.stringify(persisted)).not.toContain('127.0.0.1:7878'); - } finally { - await new Promise((resolve, reject) => { - server.close((error) => error ? reject(error) : resolve()); - }); - await rm(home, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/cli/test/helpers/live-daemon.ts b/packages/cli/test/helpers/live-daemon.ts index 217188bd0a..683b1ba5f9 100644 --- a/packages/cli/test/helpers/live-daemon.ts +++ b/packages/cli/test/helpers/live-daemon.ts @@ -89,7 +89,7 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise ({})) as { error?: string; reason?: string }; - if (body.error !== 'PublisherUnavailable' && body.error !== 'PublisherDisabled') break; + const body = await res.json().catch(() => ({})) as { code?: string; reason?: string }; + if (body.code !== 'async_publisher_unavailable') break; if (body.reason !== 'publisher_starting') { throw new Error(`Async publisher failed readiness: ${body.reason ?? res.status}`); } diff --git a/packages/cli/test/oxigraph-managed.test.ts b/packages/cli/test/oxigraph-managed.test.ts index cb89c9592f..f069eb6676 100644 --- a/packages/cli/test/oxigraph-managed.test.ts +++ b/packages/cli/test/oxigraph-managed.test.ts @@ -86,7 +86,7 @@ afterAll(async () => { describe('planManagedOxigraph', () => { it('returns null for non-oxigraph-server backends', () => { - expect(planManagedOxigraph({ store: { backend: 'oxigraph' } }, '/data')).toBeNull(); + expect(planManagedOxigraph({ store: { backend: 'oxigraph-worker' } }, '/data')).toBeNull(); expect(planManagedOxigraph({ store: { backend: 'sparql-http' } }, '/data')).toBeNull(); expect(planManagedOxigraph({}, '/data')).toBeNull(); }); @@ -327,7 +327,7 @@ describe('startManagedOxigraph (real download + real server)', () => { // Contract: a non-managed backend is a no-op — null result, and nothing // observable happens (no cache dir is created, nothing binds a port). const result = await startManagedOxigraph({ - config: { store: { backend: 'oxigraph' } }, + config: { store: { backend: 'oxigraph-worker' } }, dataDir: '/data', }); expect(result).toBeNull(); diff --git a/packages/cli/test/publisher-managed-store.test.ts b/packages/cli/test/publisher-managed-store.test.ts deleted file mode 100644 index 5e25109379..0000000000 --- a/packages/cli/test/publisher-managed-store.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { mkdtemp } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { createPublisherInspector } from '../src/publisher-runner.js'; - -describe('daemon-down publisher inspection', () => { - it('requires the daemon for the managed oxigraph-server backend', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-managed-store-')); - - await expect(createPublisherInspector({ - dataDir, - config: { - name: 'managed-publisher-test', - nodeRole: 'edge', - store: { backend: 'oxigraph-server', options: {} }, - }, - })).rejects.toThrow( - /daemon-managed store "oxigraph-server" require a running DKG daemon.*Start the daemon and retry/, - ); - }); -}); diff --git a/packages/cli/test/publisher-wallets.test.ts b/packages/cli/test/publisher-wallets.test.ts index 3b73d221b3..10e2085a9b 100644 --- a/packages/cli/test/publisher-wallets.test.ts +++ b/packages/cli/test/publisher-wallets.test.ts @@ -144,48 +144,6 @@ describe('publisher wallets', () => { ).rejects.toThrow('dkg publisher wallet add '); }); - it('boots and closes the standalone publisher runtime with the persistent fallback when config has no store block', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-runtime-')); - const wallet = ethers.Wallet.createRandom(); - await addPublisherWallet(dataDir, wallet.privateKey); - - const runtime = await createPublisherRuntime({ - dataDir, - config: { - name: 'test-node', - apiPort: 9200, - listenPort: 0, - nodeRole: 'edge', - contextGraphs: [], - chain: { type: 'mock' }, - }, - pollIntervalMs: 10, - errorBackoffMs: 10, - }); - - await runtime.publisher.lift({ - swmId: 'swm-main', - shareOperationId: 'share-no-store-fallback', - roots: ['urn:local:/fallback'], - contextGraphId: 'music-social', - namespace: 'aloha', - scope: 'person-profile', - transitionType: 'CREATE', - authority: { type: 'owner', proofRef: 'proof:owner:fallback' }, - }); - - await runtime.stop(); - const persistentStore = await createTripleStore({ - backend: 'oxigraph-persistent', - options: { path: join(dataDir, 'store.nq') }, - }); - const inspector = createPublisherInspectorFromStore(persistentStore, true); - const jobs = await inspector.publisher.list(); - expect(jobs).toHaveLength(1); - expect(jobs[0]?.jobId).toBeDefined(); - await inspector.stop(); - }); - it('resolves publisher chain defaults from config.networkConfig', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-runtime-')); const wallet = ethers.Wallet.createRandom(); diff --git a/packages/cli/test/status-route-rpc.test.ts b/packages/cli/test/status-route-rpc.test.ts index 29507ed102..ecc88c7d20 100644 --- a/packages/cli/test/status-route-rpc.test.ts +++ b/packages/cli/test/status-route-rpc.test.ts @@ -33,35 +33,13 @@ import { } from '@origintrail-official/dkg-chain'; import { computeNetworkId } from '../../core/src/genesis.js'; import { getSharedContext } from '../../chain/test/evm-test-context.js'; -import { loadNetworkConfig, type DkgConfig } from '../src/config.js'; -import { - handleStatusRoutes, - invalidateExternalStoreQuadsCache, -} from '../src/daemon/routes/status.js'; -import { - createRequestStoreContext, - type RequestContext, -} from '../src/daemon/routes/context.js'; +import { loadNetworkConfig } from '../src/config.js'; +import { handleStatusRoutes } from '../src/daemon/routes/status.js'; +import type { RequestContext } from '../src/daemon/routes/context.js'; import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from './helpers/live-daemon.js'; // A port nothing listens on — connecting to it is a REAL refused connection. const DEAD_RPC = 'http://127.0.0.1:9'; -const MANAGED_QUERY_ENDPOINT = 'http://127.0.0.1:7878/query'; - -function managedStoreRuntime(operatorConfig: DkgConfig) { - return { - operatorConfig, - effectiveStore: { backend: 'oxigraph-server', options: {} }, - runtimeStore: { - backend: 'sparql-http', - options: { - queryEndpoint: MANAGED_QUERY_ENDPOINT, - updateEndpoint: 'http://127.0.0.1:7878/update', - managedByDkg: true, - }, - }, - }; -} describe('/api/status + /api/chain/rpc-health (real daemon, real chain)', () => { let daemon: LiveDaemon; @@ -140,124 +118,9 @@ describe('/api/status + /api/chain/rpc-health (real daemon, real chain)', () => }); describe('/api/status selected overlay details', () => { - it('reports the effective managed store for a blockless config', async () => { - const config: DkgConfig = { - name: 'status-blockless-store-test', - nodeRole: 'edge', - chain: { type: 'mock' }, - }; - const query = vi.fn(async () => ({ - type: 'bindings' as const, - bindings: [{ c: '42' }], - })); - invalidateExternalStoreQuadsCache(); - const server = createServer(async (req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); - await handleStatusRoutes({ - req, - res, - path: url.pathname, - url, - network: null, - ...createRequestStoreContext(managedStoreRuntime(config)), - startedAt: Date.now(), - agent: { - peerId: 'peer-status-test', - multiaddrs: [], - node: { - libp2p: { getConnections: () => [] }, - getRelayStats: () => null, - }, - publisher: { getIdentityId: () => 0n }, - store: { query }, - }, - nodeVersion: '0.0.0-test', - nodeCommit: '', - admission: { inFlight: 0, max: 0, rejectedTotal: 0 }, - } as unknown as RequestContext); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - try { - const address = server.address() as AddressInfo; - const res = await fetch(`http://127.0.0.1:${address.port}/api/status`); - expect(res.status).toBe(200); - const body: any = await res.json(); - - expect(body.storeBackend).toBe('oxigraph-server'); - expect(body.storeUrl).toBe(MANAGED_QUERY_ENDPOINT); - expect(body.storeQuads).toBe(42); - expect(query).toHaveBeenCalledOnce(); - } finally { - invalidateExternalStoreQuadsCache(); - await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); - } - }); - - it('reports the effective managed store after an acknowledged worker cutover', async () => { - const config: DkgConfig = { - name: 'status-worker-cutover-test', - nodeRole: 'edge', - chain: { type: 'mock' }, - store: { backend: 'oxigraph-worker', options: {} }, - }; - const query = vi.fn(async () => ({ - type: 'bindings' as const, - bindings: [{ c: '17' }], - })); - invalidateExternalStoreQuadsCache(); - const server = createServer(async (req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); - await handleStatusRoutes({ - req, - res, - path: url.pathname, - url, - network: null, - ...createRequestStoreContext(managedStoreRuntime(config)), - startedAt: Date.now(), - agent: { - peerId: 'peer-status-test', - multiaddrs: [], - node: { - libp2p: { getConnections: () => [] }, - getRelayStats: () => null, - }, - publisher: { getIdentityId: () => 0n }, - store: { query }, - }, - nodeVersion: '0.0.0-test', - nodeCommit: '', - admission: { inFlight: 0, max: 0, rejectedTotal: 0 }, - } as unknown as RequestContext); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - try { - const address = server.address() as AddressInfo; - const res = await fetch(`http://127.0.0.1:${address.port}/api/status`); - expect(res.status).toBe(200); - const body: any = await res.json(); - - expect(body.storeBackend).toBe('oxigraph-server'); - expect(body.storeUrl).toBe(MANAGED_QUERY_ENDPOINT); - expect(body.storeQuads).toBe(17); - expect(query).toHaveBeenCalledOnce(); - } finally { - invalidateExternalStoreQuadsCache(); - await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); - } - }); - it('returns the network id and name for the selected overlay genesis', async () => { const network = await loadNetworkConfig('mainnet-gnosis'); expect(network).not.toBeNull(); - const config: DkgConfig = { - name: 'status-selected-overlay-test', - networkConfig: 'mainnet-gnosis', - nodeRole: 'edge', - chain: { type: 'mock' }, - }; const server = createServer(async (req, res) => { const url = new URL(req.url ?? '/', 'http://127.0.0.1'); @@ -267,7 +130,12 @@ describe('/api/status selected overlay details', () => { path: url.pathname, url, network, - ...createRequestStoreContext(managedStoreRuntime(config)), + config: { + name: 'status-selected-overlay-test', + networkConfig: 'mainnet-gnosis', + nodeRole: 'edge', + chain: { type: 'mock' }, + }, startedAt: Date.now(), agent: { peerId: 'peer-status-test', @@ -307,17 +175,6 @@ describe('/api/status selected overlay details', () => { const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const network = await loadNetworkConfig('mainnet-gnosis'); - const config: DkgConfig = { - name: 'status-failover-counter-test', - networkConfig: 'mainnet-gnosis', - nodeRole: 'edge', - chain: { - type: 'evm', - rpcUrl: 'http://127.0.0.1:9', - hubAddress: `0x${'ab'.repeat(20)}`, - chainId: 'evm:31337', - }, - }; try { // Seed the process-wide failover counters the status route reads, then // assert /api/status reflects the exact delta. This would FAIL if the @@ -338,7 +195,17 @@ describe('/api/status selected overlay details', () => { path: url.pathname, url, network, - ...createRequestStoreContext(managedStoreRuntime(config)), + config: { + name: 'status-failover-counter-test', + networkConfig: 'mainnet-gnosis', + nodeRole: 'edge', + chain: { + type: 'evm', + rpcUrl: 'http://127.0.0.1:9', + hubAddress: `0x${'ab'.repeat(20)}`, + chainId: 'evm:31337', + }, + }, startedAt: Date.now(), agent: { peerId: 'peer-status-test', diff --git a/packages/cli/test/store-backend-taxonomy.test.ts b/packages/cli/test/store-backend-taxonomy.test.ts deleted file mode 100644 index c904cb28c2..0000000000 --- a/packages/cli/test/store-backend-taxonomy.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { - STORAGE_ADAPTERS, - classifyTripleStoreBackend, - customTripleStoreBackend, - isExternalBackend, - isStorageAdapterBackend, - storageAdapterNames, -} from '@origintrail-official/dkg-storage'; -import { validateStoreConfig, type DkgConfig } from '../src/config.js'; -import { - DEFAULT_DAEMON_STORE_BACKEND, - MANAGED_DAEMON_STORE_BACKEND, - STORE_BACKENDS, - configBackendNames, - isManagedLocalBackend, - isRetiredStoreBackend, - requireStorageAdapterBackend, - storeFlagBackendNames, - storeBackendNames, - wizardBackendChoices, - type StoreBackend, -} from '../src/store-backends.js'; -import { checkExternalStoreReachable } from '../src/daemon/store-health-check.js'; -import { planManagedOxigraph } from '../src/daemon/oxigraph-managed.js'; -import { storeBackendHasStatusHealth } from '../src/daemon/routes/status.js'; - -function configForBackend(backend: StoreBackend): DkgConfig { - const policy = STORE_BACKENDS[backend]; - const options = policy.kind === 'external' - ? { [policy.queryEndpointOption]: 'http://store.test/query' } - : {}; - return { - name: 'taxonomy-test', - apiPort: 9200, - listenPort: 4001, - nodeRole: 'edge', - store: { backend, options }, - } as DkgConfig; -} - -describe('canonical store backend taxonomy', () => { - it('drives config validation and wizard discovery for every registered backend', () => { - const configBackends = configBackendNames(); - const wizardBackends = wizardBackendChoices(); - const flagBackends = storeFlagBackendNames(); - - for (const backend of storeBackendNames()) { - const policy = STORE_BACKENDS[backend]; - const errors = validateStoreConfig(configForBackend(backend)); - - expect((configBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired); - expect((wizardBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired && policy.wizard); - expect((flagBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired && policy.storeFlag); - expect(errors.some((error) => error.field === 'store.backend'), backend).toBe(policy.retired); - if (!policy.retired) expect(errors, backend).toEqual([]); - if (policy.wizard) expect('label' in policy && policy.label.length > 0, backend).toBe(true); - } - }); - - it('keeps daemon health, managed startup, and status routing aligned for every backend', async () => { - for (const backend of storeBackendNames()) { - const policy = STORE_BACKENDS[backend]; - const external = policy.kind === 'external'; - const managed = policy.kind === 'managed-local'; - - expect(isExternalBackend(backend), backend).toBe(external); - expect(isManagedLocalBackend(backend), backend).toBe(managed); - expect(isRetiredStoreBackend(backend), backend).toBe(policy.retired); - expect(isStorageAdapterBackend(backend), backend).toBe(policy.adapter); - expect(classifyTripleStoreBackend(backend).kind, backend).toBe( - policy.adapter ? 'adapter' : 'custom', - ); - expect(storeBackendHasStatusHealth(backend), backend).toBe(external || managed); - expect(planManagedOxigraph(configForBackend(backend), '/data') !== null, backend).toBe(managed); - - const fetch = vi.fn(async () => new Response('{}', { status: 200 })); - const health = await checkExternalStoreReachable({ - storeConfig: configForBackend(backend).store, - fetch, - }); - expect(health.ok, backend).toBe(true); - expect(fetch.mock.calls.length > 0, backend).toBe(external); - } - }); - - it('derives the daemon default and managed-local constant from the registry', () => { - expect(DEFAULT_DAEMON_STORE_BACKEND).toBe(MANAGED_DAEMON_STORE_BACKEND); - expect(STORE_BACKENDS[DEFAULT_DAEMON_STORE_BACKEND]).toMatchObject({ - default: true, - kind: 'managed-local', - retired: false, - }); - }); - - it('composes daemon policy over the storage-owned adapter registry', () => { - expect(storageAdapterNames()).toEqual([ - 'oxigraph', - 'oxigraph-persistent', - 'blazegraph', - 'sparql-http', - ]); - expect(storageAdapterNames()).not.toContain('oxigraph-server'); - expect(storageAdapterNames()).not.toContain('oxigraph-worker'); - for (const backend of storageAdapterNames()) { - expect(STORE_BACKENDS[backend]).toMatchObject(STORAGE_ADAPTERS[backend]); - expect(requireStorageAdapterBackend(backend)).toBe(backend); - } - expect(() => requireStorageAdapterBackend('oxigraph-server')).toThrow( - /not a constructible storage adapter/, - ); - }); - - it('requires an explicit custom-backend escape hatch outside the known registry', () => { - const backend = customTripleStoreBackend('vendor-plugin-store'); - expect(classifyTripleStoreBackend(backend)).toEqual({ - kind: 'custom', - backend: 'vendor-plugin-store', - }); - expect(() => customTripleStoreBackend('oxigraph')).toThrow(/known triple-store adapter/i); - }); -}); diff --git a/packages/cli/test/store-health-check.test.ts b/packages/cli/test/store-health-check.test.ts index 49797a2ea8..b01d80ba5a 100644 --- a/packages/cli/test/store-health-check.test.ts +++ b/packages/cli/test/store-health-check.test.ts @@ -37,7 +37,7 @@ describe('checkExternalStoreReachable', () => { it('passes through with no I/O for local backends', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await checkExternalStoreReachable({ - storeConfig: { backend: 'oxigraph' }, + storeConfig: { backend: 'oxigraph-worker' }, fetch: fn, }); expect(result.ok).toBe(true); diff --git a/packages/cli/test/store-identity-tag.test.ts b/packages/cli/test/store-identity-tag.test.ts index f438be62d2..fd8b19d354 100644 --- a/packages/cli/test/store-identity-tag.test.ts +++ b/packages/cli/test/store-identity-tag.test.ts @@ -43,7 +43,7 @@ describe('checkOrSetStoreIdentity', () => { it('skips for local oxigraph backend', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await checkOrSetStoreIdentity({ - storeConfig: { backend: 'oxigraph' }, + storeConfig: { backend: 'oxigraph-worker' }, nodeName: 'mynode', fetch: fn, }); diff --git a/packages/cli/test/store-runtime.test.ts b/packages/cli/test/store-runtime.test.ts deleted file mode 100644 index 15fbdab1b9..0000000000 --- a/packages/cli/test/store-runtime.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest'; -import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { - resolveDaemonStoreBootPlan, - resolveDaemonStoreRuntime, - type DaemonStoreBootDecision, - type DaemonStoreBootPlan, -} from '../src/daemon/store-runtime.js'; -import { saveConfig, type DkgConfig } from '../src/config.js'; -import type { StorageAdapterBackend } from '@origintrail-official/dkg-storage'; - -function mk(overrides: Partial = {}): DkgConfig { - return { - name: 'dkg-node', - apiPort: 9200, - listenPort: 4001, - nodeRole: 'edge', - ...overrides, - } as DkgConfig; -} - -function expectBootable( - decision: DaemonStoreBootDecision, -): asserts decision is DaemonStoreBootPlan { - expect(decision.kind).toBe('bootable'); - if (decision.kind !== 'bootable') { - throw new Error(`Expected a bootable store plan, got ${decision.kind}`); - } -} - -describe('resolveDaemonStoreBootPlan', () => { - afterEach(() => { - vi.unstubAllEnvs(); - }); - - it('keeps blockless operator config separate from the materialized daemon store default', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-')); - const config = mk(); - - const plan = resolveDaemonStoreBootPlan({ - config, - dataDir, - acceptStoreReset: false, - }); - - expectBootable(plan); - expect(plan.operatorConfig).toBe(config); - expect(plan.operatorConfig.store).toBeUndefined(); - expect(plan.effectiveConfig.store).toEqual({ backend: 'oxigraph-server', options: {} }); - expect(plan.effectiveStore).toEqual({ backend: 'oxigraph-server', options: {} }); - }); - - it('fails at runtime-plan construction when a managed store was not materialized', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-unmaterialized-')); - const plan = resolveDaemonStoreBootPlan({ - config: mk(), - dataDir, - acceptStoreReset: false, - }); - - expectBootable(plan); - expect(() => resolveDaemonStoreRuntime(plan, null)).toThrow( - /oxigraph-server.*not materialized to a storage adapter/, - ); - }); - - it('refines a live runtime store to a constructible adapter backend', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-adapter-')); - const plan = resolveDaemonStoreBootPlan({ - config: mk({ store: { backend: 'oxigraph', options: {} } }), - dataDir, - acceptStoreReset: false, - }); - - expectBootable(plan); - const runtime = resolveDaemonStoreRuntime(plan, null); - expect(runtime.runtimeStore.backend).toBe('oxigraph'); - expectTypeOf(runtime.runtimeStore.backend).toMatchTypeOf(); - }); - - it('does not persist the materialized store default during an unrelated config save', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-save-')); - vi.stubEnv('DKG_HOME', dataDir); - const plan = resolveDaemonStoreBootPlan({ - config: mk(), - dataDir, - acceptStoreReset: false, - }); - - expectBootable(plan); - plan.operatorConfig.sharedMemoryTtlMs = 1234; - await saveConfig(plan.operatorConfig); - - const persisted = JSON.parse(await readFile(join(dataDir, 'config.json'), 'utf8')) as DkgConfig; - expect(persisted.sharedMemoryTtlMs).toBe(1234); - expect(persisted.store).toBeUndefined(); - expect(plan.effectiveConfig.store).toEqual({ backend: 'oxigraph-server', options: {} }); - }); - - it('gates an oxigraph-server cutover when legacy store.nq exists without a backend marker', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-cutover-')); - await writeFile(join(dataDir, 'store.nq'), '

.'); - - const plan = resolveDaemonStoreBootPlan({ - config: mk({ store: { backend: 'oxigraph-server', options: {} } }), - dataDir, - acceptStoreReset: false, - }); - - expect(plan.kind).toBe('blocked-legacy-cutover'); - if (plan.kind !== 'blocked-legacy-cutover') { - throw new Error(`Expected a blocked legacy cutover, got ${plan.kind}`); - } - expect(plan.message).toContain('DKG_ACCEPT_STORE_RESET=1'); - }); - - it('does not repeatedly gate a cutover already recorded as oxigraph-server', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-recorded-')); - await writeFile(join(dataDir, 'store.nq'), '

.'); - await writeFile( - join(dataDir, '.network-state.json'), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-server', savedAt: Date.now() }), - ); - - const plan = resolveDaemonStoreBootPlan({ - config: mk(), - dataDir, - acceptStoreReset: false, - }); - - expectBootable(plan); - expect(plan.effectiveStore.backend).toBe('oxigraph-server'); - }); - - it('migrates an explicit worker config only after acknowledging its legacy store', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-worker-')); - await writeFile(join(dataDir, 'store.nq'), '

.'); - const config = mk({ store: { backend: 'oxigraph-worker' } }); - - const blocked = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: false }); - expect(blocked.kind).toBe('blocked-legacy-cutover'); - if (blocked.kind !== 'blocked-legacy-cutover') { - throw new Error(`Expected a blocked legacy cutover, got ${blocked.kind}`); - } - expect(blocked.message).toContain('legacy store.nq from the old oxigraph-worker backend'); - - const acknowledged = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: true }); - expectBootable(acknowledged); - expect(acknowledged.effectiveStore).toEqual({ backend: 'oxigraph-server', options: {} }); - expect(acknowledged.operatorConfig).toBe(config); - expect(acknowledged.operatorConfig.store?.backend).toBe('oxigraph-worker'); - }); - - it('classifies an explicit worker config without legacy data as invalid', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-invalid-worker-')); - const config = mk({ store: { backend: 'oxigraph-worker' } }); - - const decision = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: false }); - - expect(decision).toEqual({ kind: 'invalid-config', operatorConfig: config }); - }); -}); diff --git a/packages/cli/test/store-wizard.test.ts b/packages/cli/test/store-wizard.test.ts index a67557bba3..02a2ac794a 100644 --- a/packages/cli/test/store-wizard.test.ts +++ b/packages/cli/test/store-wizard.test.ts @@ -10,7 +10,7 @@ * - Blazegraph + valid URL: store block persisted with * `managedByDkg: false`. * - Blazegraph + unreachable URL: surfaces formatted failure, allows - * retry, abort leaves the managed local default in place. + * retry, abort returns to local default. * - Blazegraph + 404 URL: namespace-missing branch fires; message * mentions namespace, not network. * - Blank URL prompt: PR 2's "no Docker yet" message + retry. @@ -28,14 +28,6 @@ import { describe, it, expect } from 'vitest'; import { applyStoreFlagsToConfig, promptStoreBackend } from '../src/store-wizard.js'; import type { DkgConfig } from '../src/config.js'; -import { - STORE_BACKENDS, - configBackendList, - configBackendNames, - storeFlagBackendList, - storeFlagBackendNames, - wizardBackendChoices, -} from '../src/store-backends.js'; function mockFetch(handler: (input: any, init?: any) => Response | Promise) { const calls: Array<{ url: string; init?: RequestInit }> = []; @@ -62,26 +54,6 @@ function mockAsk(scriptedAnswers: string[]): (q: string, def?: string) => Promis // --------------------------------------------------------------------- describe('promptStoreBackend', () => { - it('keeps config, wizard, and flag backend lists aligned with their policies', () => { - expect(configBackendList().split(', ')).toEqual(configBackendNames()); - expect(configBackendNames()).toEqual([ - 'oxigraph-server', - 'oxigraph', - 'oxigraph-persistent', - 'blazegraph', - 'sparql-http', - ]); - expect(storeFlagBackendList().split(', ')).toEqual(storeFlagBackendNames()); - expect(storeFlagBackendNames()).toEqual([ - 'oxigraph-server', - 'oxigraph', - 'blazegraph', - 'sparql-http', - ]); - expect(wizardBackendChoices()).toEqual(['oxigraph-server', 'oxigraph', 'blazegraph']); - expect(Object.keys(STORE_BACKENDS)).toContain('oxigraph-worker'); - }); - it('defaults to oxigraph-server when the operator accepts the default', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await promptStoreBackend({ @@ -93,26 +65,31 @@ describe('promptStoreBackend', () => { expect(calls).toHaveLength(0); // no URL probe issued for a local backend }); - it('persists an explicit embedded development store when operator picks "oxigraph" by name', async () => { + it('returns no store block (embedded worker) when operator picks "oxigraph" by name', async () => { const result = await promptStoreBackend({ ask: mockAsk(['oxigraph']), log: () => {}, }); - expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); + expect(result.storeBlock).toBeNull(); }); - it('persists an explicit embedded development store when operator picks oxigraph by number', async () => { + it('returns no store block (embedded worker) when operator picks the worker by number', async () => { // Menu is now `1) oxigraph-server 2) oxigraph 3) blazegraph` — picking - // option 2 opts into the embedded in-memory store explicitly. + // option 2 must opt down to the embedded in-process worker (no block). const result = await promptStoreBackend({ ask: mockAsk(['2']), log: () => {}, }); - expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); + expect(result.storeBlock).toBeNull(); }); - it('preserves a supported explicit embedded backend verbatim on Enter-through (no flip, no option loss)', async () => { - for (const backend of ['oxigraph', 'oxigraph-persistent'] as const) { + it('preserves an explicit embedded backend verbatim on Enter-through (no flip, no option loss)', async () => { + // Codex #946 — only a *block-less* config should fall through to the new + // oxigraph-server default. A node that explicitly chose a local worker + // variant must keep it on a re-init Enter-through, AND keep its custom + // `options` (e.g. the worker's `options.path`): returning `null` would let + // `dkg init` write `store: undefined` and relocate the store on next boot. + for (const backend of ['oxigraph', 'oxigraph-worker', 'oxigraph-persistent'] as const) { const existingStore = { backend, options: { path: '/custom/store' } }; const result = await promptStoreBackend({ ask: mockAsk(['']), // Enter @@ -123,7 +100,11 @@ describe('promptStoreBackend', () => { } }); - it('does not preserve oxigraph-persistent options when the operator explicitly picks oxigraph', async () => { + it('switches to the default embedded worker when an oxigraph-persistent node EXPLICITLY picks oxigraph', async () => { + // Codex #946 — preservation must be gated on a true keep. An operator who + // explicitly selects option `2` / "oxigraph" to move a worker/persistent + // node back to the plain embedded default must NOT have the old backend + + // options silently retained. Both the numeric and named selection switch. const existingStore = { backend: 'oxigraph-persistent', options: { path: '/custom/store' } }; for (const answer of ['2', 'oxigraph']) { const result = await promptStoreBackend({ @@ -131,33 +112,13 @@ describe('promptStoreBackend', () => { existingStore, log: () => {}, }); - expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); + expect(result.storeBlock).toBeNull(); } }); - it('does not preserve retired oxigraph-worker configs on Enter-through', async () => { - const logs: string[] = []; - const result = await promptStoreBackend({ - ask: mockAsk(['']), - existingStore: { backend: 'oxigraph-worker', options: { path: '/custom/store' } }, - log: (m) => logs.push(m), - }); - expect(result.storeBlock).toEqual({ backend: 'oxigraph-server', options: {} }); - expect(logs.join('\n')).toMatch(/retired/); - }); - - it('rejects oxigraph-worker when typed explicitly', async () => { - await expect( - promptStoreBackend({ - ask: mockAsk(['oxigraph-worker']), - log: () => {}, - }), - ).rejects.toThrow(/no longer supported/); - }); - it('falls back to the recommended default (oxigraph-server) on an out-of-range number', async () => { // Codex #946 — a typo'd digit ("9") must not silently downgrade a fresh - // install to an embedded backend; it resolves to defaultBackend (option 1). + // install to the embedded worker; it resolves to defaultBackend (option 1). const result = await promptStoreBackend({ ask: mockAsk(['9']), log: () => {}, @@ -211,7 +172,7 @@ describe('promptStoreBackend', () => { expect(logs.some((l) => l.includes('STORE-HEALTH'))).toBe(true); }); - it('aborts to the managed local default when operator declines retry on unreachable URL', async () => { + it('aborts to local default when operator declines retry on unreachable URL', async () => { const { fn } = mockFetch(() => new Response('boom', { status: 500 })); const logs: string[] = []; const result = await promptStoreBackend({ @@ -322,7 +283,7 @@ describe('promptStoreBackend', () => { 'blazegraph', '', // blank URL 'n', // decline Docker - 'n', // decline retry-with-URL → abort to managed local default + 'n', // decline retry-with-URL → abort to local default ]), isDockerAvailable: async () => true, provisionBlazegraphDocker: async () => { @@ -655,22 +616,7 @@ describe('applyStoreFlagsToConfig', () => { storeFlag: 'neptune', log: () => {}, }), - ).rejects.toThrow(/oxigraph-server, oxigraph, blazegraph, sparql-http/); - }); - - it('does not advertise or accept oxigraph-persistent as a pathless --store choice', async () => { - const store = newMockConfig({ - ...baseConfig, - store: { backend: 'oxigraph-persistent', options: { path: '/existing/store.nq' } }, - } as DkgConfig); - const io = mockConfigIO(store); - - await expect(applyStoreFlagsToConfig({ - ...io, - storeFlag: 'oxigraph-persistent', - log: () => {}, - })).rejects.toThrow(/--store must be one of: oxigraph-server, oxigraph, blazegraph, sparql-http/); - expect(store.saved).toEqual([]); + ).rejects.toThrow(/oxigraph, blazegraph, sparql-http/); }); it('persists a daemon-managed oxigraph-server block (no URL required)', async () => { @@ -708,7 +654,7 @@ describe('applyStoreFlagsToConfig', () => { expect(store.saved[0].store).toEqual({ backend: 'oxigraph-server', options: {} }); }); - it('persists an explicit oxigraph block when --store oxigraph is passed', async () => { + it('clears existing store block when --store oxigraph is passed', async () => { const store = newMockConfig({ ...baseConfig, store: { @@ -723,10 +669,10 @@ describe('applyStoreFlagsToConfig', () => { log: () => {}, }); expect(store.saved).toHaveLength(1); - expect(store.saved[0].store).toEqual({ backend: 'oxigraph', options: {} }); + expect(store.saved[0].store).toBeUndefined(); }); - it('persists oxigraph when --store oxigraph is passed and no existing store block', async () => { + it('is a no-op when --store oxigraph is passed and no existing store block', async () => { const store = newMockConfig(baseConfig); const io = mockConfigIO(store); await applyStoreFlagsToConfig({ @@ -734,20 +680,6 @@ describe('applyStoreFlagsToConfig', () => { storeFlag: 'oxigraph', log: () => {}, }); - expect(store.saved).toHaveLength(1); - expect(store.saved[0].store).toEqual({ backend: 'oxigraph', options: {} }); - }); - - it('rejects --store oxigraph-worker', async () => { - const store = newMockConfig(baseConfig); - const io = mockConfigIO(store); - await expect( - applyStoreFlagsToConfig({ - ...io, - storeFlag: 'oxigraph-worker', - log: () => {}, - }), - ).rejects.toThrow(/no longer supported/); expect(store.saved).toEqual([]); }); diff --git a/packages/cli/test/validate-store-config.test.ts b/packages/cli/test/validate-store-config.test.ts index fefd180a61..39ff79e24c 100644 --- a/packages/cli/test/validate-store-config.test.ts +++ b/packages/cli/test/validate-store-config.test.ts @@ -8,8 +8,8 @@ * when paired with an external backend (no local store path to * infer from). * - * Supported local backends are unaffected. The retired `oxigraph-worker` - * backend is rejected before boot. + * Local backends (default Oxigraph) are unaffected — the function is a + * no-op for them. * * Plan: `.cursor/plans/blazegraph_v10_support_178da670.plan.md` §PR 1 item 6. */ @@ -33,12 +33,9 @@ describe('validateStoreConfig', () => { }); describe('local backends', () => { - it('no-op for supported local backends', () => { + it('no-op for oxigraph-worker', () => { expect( - validateStoreConfig(mk({ store: { backend: 'oxigraph' } })), - ).toEqual([]); - expect( - validateStoreConfig(mk({ store: { backend: 'oxigraph-persistent', options: { path: '/tmp/store.nq' } } })), + validateStoreConfig(mk({ store: { backend: 'oxigraph-worker' } })), ).toEqual([]); }); @@ -47,19 +44,10 @@ describe('validateStoreConfig', () => { // if the backend is local; the wipe + health check honour // isExternalBackend the same way. const errors = validateStoreConfig( - mk({ store: { backend: 'oxigraph', options: { url: 'irrelevant' } } }), + mk({ store: { backend: 'oxigraph-worker', options: { url: 'irrelevant' } } }), ); expect(errors).toEqual([]); }); - - it('rejects the retired oxigraph-worker backend', () => { - const errors = validateStoreConfig( - mk({ store: { backend: 'oxigraph-worker' } }), - ); - expect(errors).toHaveLength(1); - expect(errors[0].field).toBe('store.backend'); - expect(errors[0].message).toMatch(/no longer supported/); - }); }); describe('blazegraph', () => { @@ -148,7 +136,7 @@ describe('validateStoreConfig', () => { it('does not enforce the directory requirement for local backends', () => { const errors = validateStoreConfig( mk({ - store: { backend: 'oxigraph-persistent', options: { path: '/tmp/store.nq' } }, + store: { backend: 'oxigraph-worker' }, largeLiteralStorage: { enabled: true }, }), ); diff --git a/packages/cli/test/write-preflight-resilience.test.ts b/packages/cli/test/write-preflight-resilience.test.ts index 2e3c44dca4..3428effdca 100644 --- a/packages/cli/test/write-preflight-resilience.test.ts +++ b/packages/cli/test/write-preflight-resilience.test.ts @@ -12,9 +12,9 @@ // `contextGraphActivePublicOnChainFromRegistry` / `contextGraphExists` // methods (invoked via ContextGraphResolveMethods.prototype on a narrow // harness carrying real state, the same shape the DKGAgent mixin sees), and -// • a REAL SparqlHttpStore pointed at an unavailable loopback endpoint — the -// honest connection failure an unavailable external/managed store produces -// live — alongside a healthy embedded store for the no-change paths, and +// • a REAL OxigraphWorkerStore that has been close()d — the honest +// "the store is closed" failure a crashed/closed worker produces live — +// alongside a healthy in-memory worker store for the no-change paths, and // • the REAL `resolveRequiredWriteContextGraphId` resolver with a real // captured ServerResponse sink (same conventions as // context-graph-write-path-validation.test.ts). @@ -38,7 +38,7 @@ import { WRITE_PREFLIGHT_CHAIN_RESCUE_TIMEOUT_MS, } from '../src/daemon/http-utils.js'; import { ContextGraphResolveMethods } from '../../agent/src/dkg-agent-cg-resolve.js'; -import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; +import { OxigraphWorkerStore, createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { DKG_ONTOLOGY, SYSTEM_CONTEXT_GRAPHS, @@ -104,8 +104,8 @@ function agentHarness( // The narrow data interface resolveRequiredWriteContextGraphId consumes. // `listContextGraphs` performs a real read against the harness store so a -// unavailable store rejects with a genuine adapter error (the same failure -// mode the real list path hits), and a healthy store returns the given rows. +// closed store rejects with the genuine worker error (the same failure mode +// the real list path hits), and a healthy store returns the given rows. function providerFor(harness: any, rows: Array> = []) { const listCalls: Array = []; return { @@ -153,30 +153,25 @@ function captureRes(): { res: ServerResponse; out: { status?: number; body?: any // The exact legacy 503 body the pre-resilience code produced when both legs // threw. Live callers and tests grep for this — the rescue must leave it // byte-compatible whenever it does NOT accept. -const LEGACY_503_ERROR = /^Failed to validate contextGraphId against known context graphs: exact preflight failed: .*fetch failed.*; list validation failed: .*fetch failed/; +const LEGACY_503_ERROR = /^Failed to validate contextGraphId against known context graphs: exact preflight failed: .*store is closed.*; list validation failed: .*store is closed/; -let closedStore: TripleStore; -let healthyStore: TripleStore; +let closedStore: OxigraphWorkerStore; +let healthyStore: OxigraphWorkerStore; beforeAll(async () => { - // A real external-store adapter pointed at an unavailable endpoint: every - // query rejects through the same fetch path used by managed/external stores. - closedStore = await createTripleStore({ - backend: 'sparql-http', - options: { - queryEndpoint: 'http://127.0.0.1:65535/query', - updateEndpoint: 'http://127.0.0.1:65535/update', - }, - }); - healthyStore = await createTripleStore({ backend: 'oxigraph' }); + // A real worker-backed store that has been closed: every subsequent call + // rejects with the real `oxigraph-worker: cannot run "…" — the store is + // closed.` error — the exact live failure this track fixes. + closedStore = new OxigraphWorkerStore(undefined); + await closedStore.close(); + healthyStore = new OxigraphWorkerStore(undefined); }); afterAll(async () => { - await closedStore.close(); await healthyStore.close(); }); -describe('probeContextGraphWritePreflight — store-failure resilience (real probe, unavailable store)', () => { +describe('probeContextGraphWritePreflight — store-failure resilience (real probe, real closed store)', () => { it('detects local content from indexed graph names while ignoring bookkeeping-only graphs', async () => { const store = await createTripleStore({ backend: 'oxigraph' }); const listGraphsByPrefix = store.listGraphsByPrefix?.bind(store); @@ -235,7 +230,7 @@ describe('probeContextGraphWritePreflight — store-failure resilience (real pro ); const probe = await harness.probeContextGraphWritePreflight(CG); expect(probe.storeUnavailable).toBe(true); - expect(probe.storeErrorMessage).toMatch(/fetch failed/); + expect(probe.storeErrorMessage).toMatch(/store is closed/); // In-memory registry state needs zero store I/O and must survive. expect(probe.inMemorySubscription).toEqual({ subscribed: true, synced: true }); // Store-derived facts are UNKNOWN — a store outage must never be @@ -274,7 +269,7 @@ describe('probeContextGraphWritePreflight — store-failure resilience (real pro }); }); -describe('resolveRequiredWriteContextGraphId — both-legs-failed rescue (real resolver, unavailable store)', () => { +describe('resolveRequiredWriteContextGraphId — both-legs-failed rescue (real resolver, real closed store)', () => { it('(a) ACCEPTS on positive on-chain proof the CG is active AND PUBLIC', async () => { const isActive = recorder(async (id: bigint) => id === 7n); const getAccessPolicy = recorder(async (_id: bigint) => 0); // 0 = public diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index 3372b91cd1..9fbdf64fbf 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -87,8 +87,8 @@ export default defineConfig({ // #761 — context graph write-target validation (from main). 'test/context-graph-write-path-validation.test.ts', // Track B — write-preflight resilience when the local store is - // slow/unavailable. Real resolver + real agent probe + real - // unavailable SPARQL adapter; no hardhat needed. + // slow/closed. Real resolver + real agent probe + real (closed) + // OxigraphWorkerStore; no hardhat needed. 'test/write-preflight-resilience.test.ts', 'test/http-literal-size-validation.test.ts', // CLI subprocess smoke with stub daemon only; no hardhat needed. diff --git a/packages/core/src/proto/storage-ack.ts b/packages/core/src/proto/storage-ack.ts index d22b8d9e92..11b2fafc61 100644 --- a/packages/core/src/proto/storage-ack.ts +++ b/packages/core/src/proto/storage-ack.ts @@ -111,8 +111,8 @@ export const STORAGE_ACK_DECLINE_CODES = { /** * The core hit a peer-LOCAL, transient infrastructure failure while * servicing an otherwise well-formed request: its triple store errored - * mid-read/write (e.g. a managed or external store restarting and refusing - * a connection) or its live signer-registration chain lookup threw + * mid-read/write (e.g. an oxigraph worker mid-restart throwing + * `store is closed`) or its live signer-registration chain lookup threw * (a degraded shared RPC). Introduced after the testnet storage-ACK * dead-air incident: every such failure previously THREW out of the * handler, which ProtocolRouter's inbound wrapper surfaces as a bare @@ -151,7 +151,7 @@ export const TRANSIENT_STORAGE_ACK_DECLINE_CODES: ReadonlySet = new Set< // waiting fixes it. STORAGE_ACK_DECLINE_CODES.MISSING_CIPHERTEXT_CHUNKS, // Dead-air fix: a store/RPC blip on the core clears when the local - // local store or the shared RPC recovers — the same "wait a few + // oxigraph worker or the shared RPC recovers — the same "wait a few // seconds and re-ask" cadence as SWM catch-up. Marking it transient // keeps a briefly-degraded core in the quorum pool instead of // deselecting it on the first blip. diff --git a/packages/core/test/ensure-dkg-node-config.test.ts b/packages/core/test/ensure-dkg-node-config.test.ts index 8fc412231c..6c2d519fea 100644 --- a/packages/core/test/ensure-dkg-node-config.test.ts +++ b/packages/core/test/ensure-dkg-node-config.test.ts @@ -71,7 +71,7 @@ describe('ensureDkgNodeConfig — store-backend default (issue #960)', () => { it('does NOT flip an existing (block-less) node onto a new backend', () => { // Simulate an existing node: a config.json is already on disk (it had been - // running without an explicit store block). Re-running setup must + // running on the oxigraph-worker runtime fallback). Re-running setup must // not silently switch its backend (which would force a store reset). writeFileSync(join(tempHome, 'config.json'), JSON.stringify({ name: 'node-a', nodeRole: 'edge' }) + '\n'); ensureDkgNodeConfig({ diff --git a/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts b/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts index 3abebdcd3f..dd08785081 100644 --- a/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts +++ b/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts @@ -124,7 +124,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: true }, store: { - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', options: { path: join(home, 'store.nq') }, }, chain: { diff --git a/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts index 3732dc9d76..c057c13d7c 100644 --- a/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts @@ -17,7 +17,7 @@ * slow-RPC fault legs those PRs also fixed cannot be induced against a healthy * black-box devnet — they are pinned by the unit/integration suites shipped in * #1404 (publisher-runner-ack-readiness, policy-retry) and #1408 - * (storage-ack-core-unavailable and related store-outage scenarios). + * (storage-ack-core-unavailable, oxigraph-worker-respawn). */ import { test, expect } from '../../fixtures/base.js'; import { requireDevnetNode, requireDevnetPrecondition, waitForDevnetStatus } from '../../helpers/devnet.js'; diff --git a/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts index 0eb25f50bb..487cdaaac2 100644 --- a/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts @@ -10,9 +10,9 @@ * node does not track is NEVER admitted; refusals are structured, side-effect * free, and leave the node healthy" — is exactly what this spec pins against * the healthy devnet. (The outage-only legs — 503 fail-closed, the on-chain - * public rescue itself and store restart recovery — need a killed store and are + * public rescue itself, oxigraph worker respawn — need a killed store and are * pinned by #1408's real-component suites: write-preflight-resilience, - * storage-ack-core-unavailable and related store-outage scenarios.) + * storage-ack-core-unavailable, oxigraph-worker-respawn.) */ import { test, expect } from '../../fixtures/base.js'; import { devnetApiFetch, requireDevnetNode, requireDevnetPrecondition, waitForDevnetStatus } from '../../helpers/devnet.js'; diff --git a/packages/query/src/query-handler.ts b/packages/query/src/query-handler.ts index 4bc94d6a41..17fd97c2c0 100644 --- a/packages/query/src/query-handler.ts +++ b/packages/query/src/query-handler.ts @@ -405,7 +405,7 @@ export class QueryHandler { // post-materialization `.slice()`, so a high-cardinality query is fully // materialized before being truncated. Genuinely enforcing either bound // requires threading an `AbortSignal` + result cap through - // `QueryEngine.query()` → the selected storage adapter. Injecting a + // `QueryEngine.query()` → storage → the oxigraph worker. Injecting a // `LIMIT` into the user query here is NOT a safe shortcut: for scoped // queries it would push the statement into the multi-graph // solution-set-modifier rejection path (see #789), so it is diff --git a/packages/storage/README.md b/packages/storage/README.md index 08c2263bf4..698e3c2a05 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -6,6 +6,7 @@ Triple store abstraction layer for DKG V10. Provides a unified API over multiple - **Backend adapters** — pluggable triple store implementations: - `OxigraphStore` — embedded WASM/native store, no external dependencies + - `OxigraphWorkerStore` — worker-thread variant; keeps the daemon event loop free, with a per-read-operation timeout (see below) - `BlazegraphStore` — connects to a running Blazegraph SPARQL endpoint - `SparqlHttpStore` — generic adapter for any SPARQL 1.1 compliant endpoint - **Graph manager** — named graph lifecycle (create, drop, list) with contextGraph-scoped data and metadata graphs @@ -32,6 +33,39 @@ await store.insert(quads); const result = await store.query('SELECT * WHERE { ?s ?p ?o } LIMIT 10'); ``` +## Embedded worker store (`oxigraph-worker`) tuning + +The embedded worker runs **all** store operations on a single worker thread, so +a long-running or stuck op (a huge import, an expensive query) blocks every +other store-backed request behind it. Under real load this surfaces as the +daemon's `/api/status` staying green while `/api/query`, +`/api/context-graph/list`, and `/api/assertion/create` hang. A `store.options` +knob bounds that blast radius: + +| Option | Default | Purpose | +|---|---|---| +| `operationTimeoutMs` | `120000` | Reject a **read-only** op (`query`, `hasGraph`, `listGraphs`, `countQuads`) that exceeds this instead of hanging forever — that's where the user-visible hang shows up. `0` disables (restores unbounded behaviour). `close` is exempt — its final flush always runs to completion so shutdown can't drop pending writes. | + +Mutations (`insert`, `delete`, …) are intentionally **not** bounded by this +timeout. The bound only drops the *caller's* promise — the single worker thread +keeps running the op — so a "timed-out" write could still commit afterwards, +and the rest of the codebase treats a rejected `insert`/`delete` as a clean +failure. Bounding only reads surfaces a wedged worker on the paths that hang +without inventing an indeterminate write outcome. `insert()` therefore stays +strictly atomic (all quads commit or the call fails), which callers rely on. + +```jsonc +// ~/.dkg/config.json +"store": { + "backend": "oxigraph-worker", + "options": { "operationTimeoutMs": 120000 } +} +``` + +For heavy / production workloads, prefer an out-of-process SPARQL server +(`sparql-http` or `blazegraph`), which handles reads and writes concurrently +and keeps the daemon responsive under load. + ## Internal Dependencies - `@origintrail-official/dkg-core` — configuration types, logging, constants diff --git a/packages/storage/src/adapters/oxigraph-worker-impl.ts b/packages/storage/src/adapters/oxigraph-worker-impl.ts new file mode 100644 index 0000000000..6bfce261ce --- /dev/null +++ b/packages/storage/src/adapters/oxigraph-worker-impl.ts @@ -0,0 +1,18 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import { OxigraphStore } from './oxigraph.js'; + +const store = new OxigraphStore(workerData?.persistPath); + +parentPort!.on('message', async (msg: { id: number; method: string; args: unknown[] }) => { + try { + const fn = (store as any)[msg.method]; + if (typeof fn !== 'function') { + parentPort!.postMessage({ id: msg.id, error: `Unknown method: ${msg.method}` }); + return; + } + const result = await fn.apply(store, msg.args); + parentPort!.postMessage({ id: msg.id, result }); + } catch (err) { + parentPort!.postMessage({ id: msg.id, error: err instanceof Error ? err.message : String(err) }); + } +}); diff --git a/packages/storage/src/adapters/oxigraph-worker.ts b/packages/storage/src/adapters/oxigraph-worker.ts new file mode 100644 index 0000000000..110866bc84 --- /dev/null +++ b/packages/storage/src/adapters/oxigraph-worker.ts @@ -0,0 +1,737 @@ +import { Worker } from 'node:worker_threads'; +import { existsSync } from 'node:fs'; +import { sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { TripleStore, Quad, TripleStoreQueryOptions, QueryResult, UpdateOptions } from '../triple-store.js'; +import { registerTripleStoreAdapter } from '../triple-store.js'; +import { GraphWriteGenTracker } from '../graph-write-gen.js'; + +/** + * Default per-operation timeout for the embedded worker store. The worker is + * a SINGLE thread that processes store ops FIFO, so one slow / wedged op (a + * huge import, an expensive query, or a genuinely hung worker) blocks every + * other store-backed request queued behind it. Without a bound, the caller — + * an API route, the publisher, gossip ingest — waits FOREVER. That is the + * exact signature behind issues #997 / #999 / #1002 / #1005 / #1008: + * `/api/status` (no store) stays green while `/api/query`, + * `/api/context-graph/list`, `/api/assertion/create` never return. + * + * A bounded wait turns an indefinite hang into a surfaced error the operator + * can act on (the message points at the real fix: use an external SPARQL + * server for heavy workloads). 120s is generous enough not to trip normal + * operations yet finite. Set `operationTimeoutMs: 0` to restore the old + * unbounded behaviour. + * + * IMPORTANT — the timeout is applied to READ-ONLY ops only (see + * READ_ONLY_METHODS). A read is side-effect-free, so bounding the caller's wait + * and rejecting is always a clean, determinate failure. A mutation is NOT + * bounded: the timeout only drops the caller's promise while the single worker + * thread keeps running the op, so a "timed-out" insert/delete could STILL + * commit afterwards — and the rest of the codebase treats a rejected + * insert/delete as a clean failure, which would leave partial state visible and + * retries ambiguous. The user-visible hang in the issues above is on the read + * paths (`/api/query`, `/api/context-graph/list`); bounding reads surfaces a + * wedged worker there without inventing an indeterminate mutation outcome. + */ +const DEFAULT_OPERATION_TIMEOUT_MS = 120_000; + +/** + * Backoff (ms) before each consecutive respawn attempt after an UNEXPECTED + * worker exit: the first attempt is immediate (a one-off OOM/crash should + * recover with zero visible downtime), then 1s / 5s / 30s, capped at the last + * tier. The cap keeps a persistently-crashing worker (corrupt state, chronic + * OOM on load) from melting the node in a hot spawn/crash loop while still + * retrying often enough that a transient cause self-heals. + */ +const RESPAWN_BACKOFF_MS = [0, 1_000, 5_000, 30_000]; + +/** + * Give-up bound: after this many CONSECUTIVE respawned workers die without + * serving a single successful op, stop respawning and latch the store closed + * (the pre-recovery behaviour) with fatal operator guidance. The counter + * resets on the first successful reply from a worker (see the message handler + * in spawnWorker), so occasional crashes days apart never accumulate here — + * only a genuine crash loop trips it. + */ +const MAX_CONSECUTIVE_RESPAWNS = 5; + +/** Unref'd sleep — a respawn backoff timer must not keep the process alive on its own. */ +function sleep(ms: number): Promise { + return new Promise((resolve) => { + const t = setTimeout(resolve, ms); + if (typeof t.unref === 'function') t.unref(); + }); +} + +export interface OxigraphWorkerStoreOptions { + /** + * Per-operation timeout in milliseconds for READ-ONLY ops. Default 120_000. + * 0 disables it. Mutations are intentionally never bounded (see + * DEFAULT_OPERATION_TIMEOUT_MS) so a timed-out write can't be reported as a + * clean failure while it is still in flight. + */ + operationTimeoutMs?: number; +} + +/** + * Accept only a finite, non-negative override; otherwise fall back. The result + * is floored to an INTEGER — the timeout is a millisecond count, so a fractional + * value is meaningless noise. + */ +function normalizeNonNegativeInt(value: number | undefined, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 + ? Math.floor(value) + : fallback; +} + +function asAbortError(reason: unknown): Error { + return reason instanceof Error ? reason : new Error(String(reason ?? 'aborted')); +} + +/** + * Side-effect-free read methods. ONLY these are bounded by the per-op timeout: + * rejecting a read after the bound is a clean, determinate failure (nothing was + * written), so it safely surfaces a wedged worker on the exact paths that hang + * in production. Every other method mutates persisted state and is left + * unbounded — see DEFAULT_OPERATION_TIMEOUT_MS for why timing out a mutation + * would be unsafe with the current call sites. + */ +const READ_ONLY_METHODS = new Set([ + 'query', 'hasGraph', 'listGraphs', 'countQuads', +]); + +/** Rejection raised when a read op exceeds its per-op timeout. */ +export interface OxigraphWorkerTimeoutError extends Error { + code: 'OXIGRAPH_WORKER_OP_TIMEOUT'; + /** Which store method timed out. */ + method: string; + /** The bound that was exceeded. */ + timeoutMs: number; +} + +/** + * Explicit worker POLICY state, replacing the old cluster of interdependent + * booleans/promises that all had to agree. `respawnGaveUp` folds into the + * 'gave_up' state and the new terminal 'in_memory_lost' state; `workerExited` + * stays as a separate lower-level FACT (it is orthogonal — see the field + * comment), and `closePromise`/`respawnPromise`/`consecutiveRespawnFailures` + * still carry their own orthogonal data. A single discriminated field makes the + * invalid states that used to be representable — e.g. "gave up" AND "live", or + * an in-memory store silently marked healthy after a crash — impossible to + * construct, and it gives every read a single, obvious source of truth. The + * state graph is: + * + * live ──unexpected exit, persisted──▶ respawning ──spawned──▶ live + * live ──unexpected exit, in-memory──▶ in_memory_lost (terminal — finding 1) + * respawning ──crash-loop bound hit──▶ gave_up (terminal) + * live | respawning ──close()────────▶ closing ──drained──▶ closed (terminal) + * + * `respawning` is the only state in which the current worker thread is dead but + * a replacement is on the way, so parked ops wait it out (see callAfterRespawn). + * Every other non-`live` state means "no usable worker" and fails ops fast. + */ +type WorkerLifecycle = + | 'initializing' + | 'live' + | 'respawning' + | 'closing' + | 'closed' + | 'gave_up' + | 'in_memory_lost'; + +/** Terminal states: the store will never serve another op and never respawns. */ +const TERMINAL: ReadonlySet = new Set([ + 'closed', 'gave_up', 'in_memory_lost', +]); + +export class OxigraphWorkerStore implements TripleStore { + readonly queryCancellation = 'interruptible' as const; + + // Assigned by spawnWorker(), which the constructor always calls — hence the + // definite-assignment assertion instead of an initializer. + private worker!: Worker; + private nextId = 0; + private pending = new Map void; reject: (e: Error) => void }>(); + // #1609: per-graph write generations, bumped client-side after each + // successful mutation RPC (the worker owns no caller-visible caches). + // Feeds the chain-reconcile negative memo via `asGraphWriteGenSource`. + private readonly writeGen = new GraphWriteGenTracker(); + private readonly operationTimeoutMs: number; + /** Resolved path of the compiled worker impl; reused verbatim on respawn. */ + private readonly workerPath: string; + /** Persistence file handed to every spawned worker (undefined = in-memory). */ + private readonly persistPath: string | undefined; + /** + * Single source of truth for the store's high-level POLICY state (see + * WorkerLifecycle). It subsumes the former `respawnGaveUp` boolean (⇔ state + * === 'gave_up') and adds the terminal 'in_memory_lost' state that finding-1 + * needs, so the give-up/close/data-lost verdicts can never disagree the way + * an ad-hoc cluster of interdependent booleans could. Set to 'live' the + * moment spawnWorker() arms a fresh thread; only ever moves along the + * documented transitions (every write is guarded — `setLifecycle` for the + * non-spawn hops and `markSpawnedLive` for the spawn → 'live' hop — so a + * terminal state can never silently reopen). + * + * `workerExited` below stays as a separate, lower-level FACT ("the current + * thread has exited") because it is genuinely orthogonal to policy: during a + * graceful close() the state is 'closing' while the thread is still alive and + * mid-flush, then the thread exits — same policy state, different fact. The + * old soup was the *interdependence* of workerExited + respawnGaveUp + + * respawnPromise + closePromise all having to agree; folding the policy half + * into one discriminated field removes that. + * + * The initializer is a pre-construction placeholder only: the constructor + * always calls spawnWorker() (whose `markSpawnedLive` accepts the + * non-terminal 'initializing' placeholder), so no op ever observes this + * value. 'initializing' is deliberately NON-terminal so the spawn guard can + * enforce terminal permanence uniformly — a placeholder of 'closed' (a + * terminal state) would have made the first legal spawn indistinguishable + * from illegally reopening a closed store. + */ + private lifecycle: WorkerLifecycle = 'initializing'; + /** Set once the CURRENT worker thread has exited (graceful close, crash, or kill); cleared by spawnWorker(). */ + private workerExited = false; + /** Memoized close so repeat/concurrent close() calls share one teardown. */ + private closePromise: Promise | null = null; + /** + * In-flight replacement of a crashed worker. While set, NEW ops park on it + * (see callWithTimeout) instead of failing, so a request that races the + * respawn sees a slightly slower success rather than a spurious error. + * Null whenever a live worker is armed. Tracked separately from `lifecycle` + * because ops need the actual promise to await, not just the 'respawning' tag. + */ + private respawnPromise: Promise | null = null; + /** + * How many respawned workers in a row died without answering a single op + * successfully. Reset to 0 by the first successful reply; feeds the backoff + * tier and the MAX_CONSECUTIVE_RESPAWNS give-up bound. + */ + private consecutiveRespawnFailures = 0; + + constructor(persistPath?: string, opts?: OxigraphWorkerStoreOptions) { + this.operationTimeoutMs = normalizeNonNegativeInt(opts?.operationTimeoutMs, DEFAULT_OPERATION_TIMEOUT_MS); + + // Resolve the worker impl with a small search path so this keeps + // working in all three deployment shapes we actually run in: + // + // 1. Production / npm install / built monorepo — this module is + // loaded from `dist/adapters/oxigraph-worker.js`, so the + // sibling `./oxigraph-worker-impl.js` resolves correctly. + // 2. vitest against raw source — this module is loaded from + // `src/adapters/oxigraph-worker.ts`, so the sibling + // `./oxigraph-worker-impl.js` does NOT exist, but its compiled + // twin in `dist/adapters/` does as long as the caller ran + // `pnpm --filter ...dkg-storage build` first. Redirect to + // that path so the adapter is runnable in dev loops. + // 3. Neither file exists — genuinely unbuilt tree. Throw a loud, + // actionable error explaining the fix (`pnpm build`), matching + // the expectation in `test/storage.test.ts`. + const siblingJsUrl = new URL('./oxigraph-worker-impl.js', import.meta.url); + const siblingJsPath = fileURLToPath(siblingJsUrl); + let workerPath: string | null = existsSync(siblingJsPath) ? siblingJsPath : null; + if (!workerPath) { + const srcAdapters = `${sep}src${sep}adapters${sep}`; + const distAdapters = `${sep}dist${sep}adapters${sep}`; + if (siblingJsPath.includes(srcAdapters)) { + const candidate = siblingJsPath.replace(srcAdapters, distAdapters); + if (existsSync(candidate)) workerPath = candidate; + } + } + if (!workerPath) { + throw new Error( + `oxigraph-worker adapter: compiled worker artefact ` + + `\`oxigraph-worker-impl.js\` was not found next to ` + + `${siblingJsPath} or in the sibling \`dist/adapters/\` ` + + `directory. Run \`pnpm --filter @origintrail-official/dkg-storage build\` ` + + `before using this adapter.`, + ); + } + this.workerPath = workerPath; + this.persistPath = persistPath; + this.spawnWorker(); + } + + /** + * The single writer for `this.lifecycle`. A typed setter (rather than raw + * field assignment scattered across the class) gives us one place to assert + * the invariant that matters: a TERMINAL state is a dead end. Without this + * guard a latent bug — a stray respawn resurrecting a `gave_up` store, or a + * crash handler firing after `close()` — could silently re-open a store the + * operator was told is gone, which is exactly the class of "healthy-looking + * but wrong" state finding 1 is about. If we ever try to leave a terminal + * state we throw loudly instead. + */ + private setLifecycle(next: WorkerLifecycle): void { + if (this.lifecycle === next) return; + if (TERMINAL.has(this.lifecycle)) { + throw new Error( + `oxigraph-worker: illegal lifecycle transition ${this.lifecycle} → ${next} ` + + '(terminal states are permanent) — this is a bug in the respawn supervisor.', + ); + } + this.lifecycle = next; + } + + /** + * The ONE guarded transition INTO 'live' — used by spawnWorker() for both the + * constructor's first spawn (from the 'initializing' placeholder) and a + * respawn (from 'respawning'). `setLifecycle` can't own this hop because + * entering 'live' is legal only from those two non-terminal states, but the + * terminal-permanence invariant must still hold: a spawn attempted after + * 'closed' / 'gave_up' / 'in_memory_lost' (e.g. a future respawn/close code + * path calling spawnWorker() by mistake) MUST throw rather than silently + * resurrect a store that promised it would never serve another op. + */ + private markSpawnedLive(): void { + if (this.lifecycle === 'live') return; + if (this.lifecycle !== 'initializing' && this.lifecycle !== 'respawning') { + throw new Error( + `oxigraph-worker: illegal spawn transition ${this.lifecycle} → live ` + + '(a worker may only be spawned from initializing or respawning; ' + + 'terminal states are permanent) — this is a bug in the respawn supervisor.', + ); + } + this.lifecycle = 'live'; + } + + /** True once an operator-initiated close() has begun (closing or closed). */ + private get isClosing(): boolean { + return this.lifecycle === 'closing' || this.lifecycle === 'closed'; + } + + /** + * Create the worker thread and arm its lifecycle handlers. Shared by the + * constructor and by respawn() so a replacement worker is wired IDENTICALLY + * to the original — same impl path, same workerData, same handlers — and the + * OxigraphWorkerStore object keeps its identity, so every alias held across + * the daemon (routes, publisher, gossip ingest) transparently talks to the + * new thread. Each handler captures the worker it was armed for and no-ops + * if `this.worker` has since moved on, so a stale event from an + * already-replaced thread can never clobber the live one's state. + */ + private spawnWorker(): void { + const worker = new Worker(this.workerPath, { + workerData: { persistPath: this.persistPath }, + }); + this.worker = worker; + this.workerExited = false; + // Arming a fresh thread IS the transition into 'live'. Route it through the + // guarded `markSpawnedLive` (not `setLifecycle`, which forbids ALL entries + // to 'live'): it permits the two legal predecessors — the 'initializing' + // placeholder (constructor's first spawn) and 'respawning' (a respawn) — + // while still throwing on terminal states, so a stray spawn after close / + // give-up / in-memory-loss can never silently resurrect the store. close() + // and the respawn supervisor are the only other writers and never race + // this: a spawn only happens in the constructor or within respawn(), which + // bails the moment a close() is seen. + this.markSpawnedLive(); + worker.on('message', (msg: { id: number; result?: unknown; error?: string }) => { + if (this.worker !== worker) return; + // Any successful reply proves this worker is healthy, which ends the + // crash-loop accounting window (see MAX_CONSECUTIVE_RESPAWNS). + if (!msg.error) this.consecutiveRespawnFailures = 0; + const p = this.pending.get(msg.id); + if (!p) return; + this.pending.delete(msg.id); + if (msg.error) p.reject(new Error(msg.error)); + else p.resolve(msg.result); + }); + worker.on('error', (err) => { + if (this.worker !== worker) return; + // Loud by design. 'error' means the worker thread threw an uncaught + // exception (an 'exit' event follows). This used to be silent, which is + // how a testnet node served HTTP for DAYS with a dead store — green + // /api/status, 503 on every write — and nothing in the logs said why. + console.error('[oxigraph-worker] worker thread error (thread is going down):', err); + for (const [, p] of this.pending) p.reject(err); + this.pending.clear(); + }); + // Once the worker exits, no pending op can ever get a reply. Settle them all + // (reject) instead of leaving callers hung forever — this is what makes the + // unbounded `close()` safe: if a second close (or any op) is still queued + // when terminate() kills the thread, it rejects here rather than hanging. + // + // Three very different exits land here (branch on lifecycle + persistPath): + // • intentional — close() ran (state is 'closing'). Stay closed; close() + // owns the final transition to 'closed'. + // • unexpected, disk-persisted — the thread died on its own. + // ERR_WORKER_OUT_OF_MEMORY, for example, kills ONLY the worker thread, + // not the process, so the daemon would otherwise keep serving with a + // permanently dead store. The committed state is on disk, so a fresh + // worker on the same path reopens it — go 'respawning' and recover. + // • unexpected, IN-MEMORY — there is no disk to reload from, so a fresh + // worker would come up EMPTY and look perfectly healthy while every + // row written before the crash is silently gone (confirmed live: data + // vanished). That is unrecoverable, so we FAIL CLOSED into the terminal + // 'in_memory_lost' state instead of respawning an empty store — every + // later op then rejects with a clear data-loss error (see postToWorker) + // rather than returning wrong-but-plausible results. + worker.on('exit', (code) => { + if (this.worker !== worker) return; + this.workerExited = true; + const intentional = this.isClosing; + // Distinguish the two unexpected exits up front so the log and the + // lifecycle transition below agree on what just happened. + const inMemoryLost = !intentional && this.persistPath === undefined; + if (intentional) { + console.info(`[oxigraph-worker] worker exited (code ${code}) — initiated by close()`); + } else if (inMemoryLost) { + console.error( + `[oxigraph-worker] worker exited UNEXPECTEDLY (code ${code}) — nobody called close(). ` + + 'FATAL: this store is IN-MEMORY (no persistPath), so its entire contents were lost with the ' + + 'worker thread and CANNOT be recovered. Failing the store closed instead of silently continuing ' + + 'with an empty store — every store-backed request will now fail fast. Use a disk-persisted store ' + + '(store.path) if the workload must survive a worker crash.', + ); + } else { + console.error( + `[oxigraph-worker] worker exited UNEXPECTEDLY (code ${code}) — nobody called close(). ` + + 'The store is disk-persisted; respawning a fresh worker on the same path.', + ); + } + if (this.pending.size > 0) { + const err = intentional + ? new Error('oxigraph-worker: worker exited before the operation completed (store closed)') + : inMemoryLost + ? new Error( + `oxigraph-worker: the IN-MEMORY worker exited unexpectedly (code ${code}) before the operation ` + + 'completed and its data was lost — an in-memory store cannot be recovered from a worker crash.', + ) + : new Error( + `oxigraph-worker restarted — retry: the worker exited unexpectedly (code ${code}) before the ` + + 'operation completed, so its outcome is unknown; a replacement worker is being spawned.', + ); + for (const [, p] of this.pending) p.reject(err); + this.pending.clear(); + } + // Route the state transition. close() already moved us to 'closing' and + // owns the final hop to 'closed', so the intentional case touches nothing. + if (inMemoryLost) { + this.setLifecycle('in_memory_lost'); + } else if (!intentional) { + this.scheduleRespawn(); + } + }); + } + + /** Arm (at most one) background respawn; the policy lives in respawn(). */ + private scheduleRespawn(): void { + // Never resurrect a store that has already given up, been closed, or lost + // its in-memory data, and never stack a second supervisor on an in-flight + // one. This is only ever reached from the exit handler's disk-persisted + // unexpected-exit branch, so we're normally transitioning out of 'live'. + if (this.respawnPromise || TERMINAL.has(this.lifecycle)) return; + this.setLifecycle('respawning'); + // respawn() never rejects (every failure path is caught and looped or + // latched), so this promise can't become an unhandled rejection while no + // op happens to be parked on it. + this.respawnPromise = this.respawn().finally(() => { + this.respawnPromise = null; + }); + } + + /** + * Replace a crashed worker with a fresh one: immediate first attempt, then + * capped backoff (RESPAWN_BACKOFF_MS) between consecutive attempts, giving + * up for good after MAX_CONSECUTIVE_RESPAWNS workers in a row die without + * serving a single successful op. Giving up latches the store closed — + * exactly the pre-recovery behaviour — but with a FATAL log telling the + * operator what happened and where to look, instead of the old silence. + */ + private async respawn(): Promise { + while (true) { + // close() raced the respawn — honour it. An operator-initiated close + // must never be resurrected by the supervisor. (close() flips lifecycle + // to 'closing' atomically before awaiting, so isClosing is the signal.) + if (this.isClosing) return; + if (this.consecutiveRespawnFailures >= MAX_CONSECUTIVE_RESPAWNS) { + this.setLifecycle('gave_up'); + console.error( + `[oxigraph-worker] FATAL: the worker died ${MAX_CONSECUTIVE_RESPAWNS} times in a row without ` + + 'serving a single successful operation — giving up on respawn and latching the store closed. ' + + 'Every store-backed request will now fail fast. Investigate the crash cause (worker OOM → raise ' + + 'memory or move to an external SPARQL backend via store.backend "sparql-http" / "blazegraph"; ' + + 'corrupt persist file → see the quarantine log) and restart the node.', + ); + return; + } + const attempt = this.consecutiveRespawnFailures; + this.consecutiveRespawnFailures += 1; + const delayMs = RESPAWN_BACKOFF_MS[Math.min(attempt, RESPAWN_BACKOFF_MS.length - 1)]; + if (delayMs > 0) { + console.error( + `[oxigraph-worker] respawn attempt ${attempt + 1}/${MAX_CONSECUTIVE_RESPAWNS} in ${delayMs}ms…`, + ); + await sleep(delayMs); + // Re-check: close() may have arrived during the backoff sleep. + if (this.isClosing) return; + } + try { + this.spawnWorker(); + console.info(`[oxigraph-worker] respawned worker (attempt ${attempt + 1}/${MAX_CONSECUTIVE_RESPAWNS})`); + return; + } catch (err) { + // new Worker() itself can throw (e.g. the impl file vanished at + // runtime). Treat it exactly like a worker that died instantly and + // loop into the next backoff tier. + console.error('[oxigraph-worker] respawn attempt failed to start a worker:', err); + } + } + } + + private call(method: string, ...args: unknown[]): Promise { + // Only read-only ops are bounded; mutations run unbounded (timeoutMs 0) so a + // timed-out write is never reported as a clean failure while still in flight. + const timeoutMs = READ_ONLY_METHODS.has(method) ? this.operationTimeoutMs : 0; + return this.callWithTimeout(timeoutMs, undefined, method, ...args); + } + + /** + * Post one op to the (live) worker and await its reply, bounding the caller's + * wait by `timeoutMs` (0 = wait indefinitely). The bound is per-CALLER: on + * timeout or caller abort we reject and drop the pending entry, but the + * single-threaded worker is STILL running the op — the late reply is then + * ignored (the message handler no-ops on a missing id) rather than + * double-settling this promise. Only read-only ops are ever given a non-zero + * timeout (see `call`), so a fired timeout is always a determinate, + * side-effect-free failure. If a crashed worker is being replaced, the op + * first parks on the respawn (the timeout starts only once it is actually + * posted — backoff is capped well below the default read bound anyway). + */ + private callWithTimeout( + timeoutMs: number, + signal: AbortSignal | undefined, + method: string, + ...args: unknown[] + ): Promise { + // A crashed worker may be mid-replacement right now. Park the op on the + // respawn instead of failing it: the store is disk-persisted and about to + // come back, so the caller sees a slightly slower success rather than a + // spurious "store is closed" for a condition the store is already fixing. + if (this.respawnPromise) return this.callAfterRespawn(timeoutMs, signal, method, args); + return this.postToWorker(timeoutMs, signal, method, args); + } + + /** + * Wait out the in-flight respawn(s), then post as normal. A loop rather + * than a single await because the replacement worker can itself die before + * this op gets posted, arming a new respawnPromise. If the respawn gave up, + * postToWorker's workerExited guard turns this into the fail-fast closed + * error (with the crash-loop guidance appended). + */ + private async callAfterRespawn( + timeoutMs: number, + signal: AbortSignal | undefined, + method: string, + args: unknown[], + ): Promise { + while (this.respawnPromise) await this.respawnPromise; + return this.postToWorker(timeoutMs, signal, method, args); + } + + private postToWorker( + timeoutMs: number, + signal: AbortSignal | undefined, + method: string, + args: unknown[], + ): Promise { + const id = this.nextId++; + return new Promise((resolve, reject) => { + // The worker is gone (closed, crashed with respawn given up, or an + // in-memory store whose data was lost) — a posted message would never be + // answered, so fail fast instead of registering a pending entry that can + // only ever hang. The lifecycle picks the RIGHT diagnostic: + // • in_memory_lost — the data is unrecoverable; say so plainly instead + // of pretending the store merely "closed" (finding 1: never let an + // in-memory crash look like a clean close or an empty-but-healthy store). + // • gave_up — closed + the crash-loop guidance. + // • otherwise — a plain operator close. + if (this.workerExited) { + if (this.lifecycle === 'in_memory_lost') { + reject(new Error( + `oxigraph-worker: cannot run "${method}" — the IN-MEMORY store's worker crashed and its data was ` + + 'lost. An in-memory store cannot be recovered from a worker crash; every request now fails ' + + 'fast. Use a disk-persisted store (store.path) or restart the node to start from empty.', + )); + return; + } + reject(new Error( + `oxigraph-worker: cannot run "${method}" — the store is closed.` + + (this.lifecycle === 'gave_up' + ? ' (The worker crashed repeatedly and automatic respawn gave up — restart the node and ' + + 'investigate the [oxigraph-worker] crash logs.)' + : ''), + )); + return; + } + if (signal?.aborted) { + reject(asAbortError(signal.reason)); + return; + } + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const cleanup = () => { + if (timer) clearTimeout(timer); + if (signal && onAbort) signal.removeEventListener('abort', onAbort); + }; + if (timeoutMs > 0) { + timer = setTimeout(() => { + if (this.pending.delete(id)) { + cleanup(); + const err = new Error( + `oxigraph-worker: "${method}" timed out after ${timeoutMs}ms. ` + + `The embedded store runs on a single worker thread, so a long-running or ` + + `stuck operation blocks all reads queued behind it. For heavy workloads ` + + `point the node at an external SPARQL server (store.backend "sparql-http" ` + + `/ "blazegraph"), or raise / disable store.options.operationTimeoutMs.`, + ) as OxigraphWorkerTimeoutError; + err.code = 'OXIGRAPH_WORKER_OP_TIMEOUT'; + err.method = method; + err.timeoutMs = timeoutMs; + reject(err); + } + }, timeoutMs); + // A pending-op timer must not keep the process alive on its own. + if (typeof timer.unref === 'function') timer.unref(); + } + this.pending.set(id, { + resolve: (v) => { cleanup(); resolve(v as T); }, + reject: (e) => { cleanup(); reject(e); }, + }); + if (signal) { + onAbort = () => { + if (this.pending.delete(id)) { + cleanup(); + reject(asAbortError(signal.reason)); + } + }; + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) { + onAbort(); + return; + } + } + try { + this.worker.postMessage({ id, method, args }); + } catch (err) { + if (this.pending.delete(id)) { + cleanup(); + reject(err instanceof Error ? err : new Error(String(err))); + } + } + }); + } + + // A single atomic worker message: all quads commit together or the call + // fails. The contract is "all-or-nothing" and every caller can rely on it + // (e.g. FinalizationHandler packs both canonical copies into one insert so one + // can't land without the other). Large idempotent bulk imports that want + // head-of-line fairness should run on an external SPARQL server, not by + // silently fragmenting this insert into non-atomic chunks. + async insert(quads: Quad[]): Promise { + await this.call('insert', quads); + this.writeGen.recordGraphWrites(new Set(quads.map((q) => q.graph || ''))); + } + async delete(quads: Quad[]): Promise { + await this.call('delete', quads); + this.writeGen.recordGraphWrites(new Set(quads.map((q) => q.graph || ''))); + } + async deleteByPattern(pattern: Partial): Promise { + const removed = await this.call('deleteByPattern', pattern); + if (pattern.graph) this.writeGen.recordGraphWrites([pattern.graph]); + else this.writeGen.recordUnscopedWrite(); + return removed; + } + // Server-side SPARQL UPDATE forwarded to the worker's OxigraphStore (which + // implements `update`); same atomic single-message contract as `insert`. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async update(sparql: string, _options?: UpdateOptions): Promise { + await this.call('update', sparql); + // A raw UPDATE's write scope is not derivable at the call site + // (`touchedGraphs` hints only membership changes) — unscoped bump. + this.writeGen.recordUnscopedWrite(); + } + async query(sparql: string, options?: TripleStoreQueryOptions): Promise { + return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'query', sparql); + } + async hasGraph(graphUri: string, options?: TripleStoreQueryOptions): Promise { + return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'hasGraph', graphUri); + } + async createGraph(graphUri: string): Promise { return this.call('createGraph', graphUri); } + async dropGraph(graphUri: string): Promise { + await this.call('dropGraph', graphUri); + this.writeGen.recordGraphWrites([graphUri]); + } + async listGraphs(options?: TripleStoreQueryOptions): Promise { + return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'listGraphs'); + } + async deleteBySubjectPrefix(graphUri: string, prefix: string): Promise { + const removed = await this.call('deleteBySubjectPrefix', graphUri, prefix); + this.writeGen.recordGraphWrites([graphUri]); + return removed; + } + + /** {@link GraphWriteGenSource} capability (#1609) — see graph-write-gen.ts. */ + getWriteGen(graphPrefix: string): number { + return this.writeGen.getWriteGen(graphPrefix); + } + async countQuads(graphUri?: string): Promise { return this.call('countQuads', graphUri); } + async flush(): Promise { return this.call('flush'); } + + async close(): Promise { + // Memoized + serialized: every close() call shares ONE teardown promise, so + // we issue exactly one close RPC. (A second unbounded close RPC would be + // orphaned when terminate() kills the worker after the first resolves, and + // — without the exit handler — would hang forever.) + // + // Flip the lifecycle to 'closing' FIRST (before awaiting anything), unless + // the store already reached a terminal state on its own (gave_up / + // in_memory_lost, or an even earlier close). Doing it synchronously here is + // what lets an in-flight respawn see the close and bail (respawn() checks + // isClosing) and what makes the worker's 'exit' handler treat this exit as + // intentional. A store that already latched terminal is simply reaped below + // — its state is permanent, so we must NOT try to move it to 'closing'. + if (!this.closePromise) { + if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closing'); + this.closePromise = this.doClose(); + } + return this.closePromise; + } + + private async doClose(): Promise { + // Worker already gone (crash/kill, respawn give-up, or in-memory loss) — + // there's nothing to flush; just make sure the thread is reaped. Keeps + // close() idempotent and non-throwing. Only settle into the terminal + // 'closed' state when we're not already in a *different* terminal state + // (gave_up / in_memory_lost stay as-is so their diagnostics survive). + if (this.workerExited) { + try { await this.worker.terminate(); } catch { /* already terminated */ } + if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closed'); + return; + } + // `close` runs the worker's FINAL synchronous flush (insert() only schedules + // a 50ms debounced flush, so close is what guarantees durability). It is + // therefore EXEMPT from the per-op timeout (timeoutMs 0): bounding it could + // fire the timeout while the worker is mid-flush, and the `finally` would + // then terminate() the thread before pending writes hit disk — losing data. + // terminate() always runs in `finally`; the worker's 'exit' handler then + // rejects anything still pending, so nothing leaks or hangs. + try { + await this.callWithTimeout(0, undefined, 'close'); + } finally { + await this.worker.terminate(); + // The 'exit' handler above left us in 'closing' (intentional exit). Land + // the terminal transition here so a post-close op fails fast (workerExited + // is now set, and the guard reads 'closed' for the plain message). + if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closed'); + } + } +} + +registerTripleStoreAdapter('oxigraph-worker', async (opts) => { + const filePath = opts?.path as string | undefined; + return new OxigraphWorkerStore(filePath, { + operationTimeoutMs: + typeof opts?.operationTimeoutMs === 'number' ? (opts.operationTimeoutMs as number) : undefined, + }); +}); diff --git a/packages/storage/src/adapters/oxigraph.ts b/packages/storage/src/adapters/oxigraph.ts index d5c9aa8522..f476b3176d 100644 --- a/packages/storage/src/adapters/oxigraph.ts +++ b/packages/storage/src/adapters/oxigraph.ts @@ -254,8 +254,8 @@ export class OxigraphStore implements TripleStore { async query(sparql: string, options?: TripleStoreQueryOptions): Promise { throwIfAborted(options?.signal); // The embedded Oxigraph binding executes synchronously, so a caller abort - // cannot interrupt this native call mid-flight. Use an HTTP backend when - // long sync queries need prompt cancellation. + // cannot interrupt this native call mid-flight. Use oxigraph-worker or an + // HTTP backend when long sync queries need prompt cancellation. const result = this.store.query(sparql); throwIfAborted(options?.signal); diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index c916a1843f..5e8f4ee97c 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -10,35 +10,16 @@ export { type AskResult, type TripleStoreConfig, type TripleStoreBackend, - type TripleStoreFactoryConfig, type TripleStoreQueryOptions, type UpdateOptions, type LargeLiteralStorageConfig, registerTripleStoreAdapter, createTripleStore, - toTripleStoreBackend, tryUpdateWithTouchedGraphs, + isExternalBackend, getSparqlEndpoint, type SparqlEndpoint, - type SparqlEndpointStoreConfig, } from './triple-store.js'; -export { - STORAGE_ADAPTERS, - classifyTripleStoreBackend, - customTripleStoreBackend, - getStorageAdapterPolicy, - isExternalBackend, - isStorageAdapterBackend, - storageAdapterNames, - type ClassifiedTripleStoreBackend, - type CustomTripleStoreBackend, - type ExternalStoreBackend, - type LocalStoreBackend, - type StorageAdapterBackend, - type StorageAdapterKind, - type StorageAdapterOfKind, - type StorageAdapterPolicy, -} from './store-backends.js'; export { StorePriorityScheduler, externalStorePriorityScheduler, @@ -79,6 +60,7 @@ export { } from './graph-write-gen.js'; export { OxigraphStore } from './adapters/oxigraph.js'; +export { OxigraphWorkerStore } from './adapters/oxigraph-worker.js'; export { BlazegraphStore } from './adapters/blazegraph.js'; export { SparqlHttpStore, @@ -105,5 +87,6 @@ export { PrivateContentStore } from './private-store.js'; // Side-effect: register built-in adapters import './adapters/oxigraph.js'; +import './adapters/oxigraph-worker.js'; import './adapters/blazegraph.js'; import './adapters/sparql-http.js'; diff --git a/packages/storage/src/store-backends.ts b/packages/storage/src/store-backends.ts deleted file mode 100644 index 0d07ba1b2b..0000000000 --- a/packages/storage/src/store-backends.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Canonical metadata for adapters the storage factory can construct. - * - * Daemon defaults, retired config names, migration policy, and CLI labels live - * in the CLI package. Keeping this registry adapter-only prevents the storage - * layer from acquiring daemon lifecycle or presentation policy. - */ -export const STORAGE_ADAPTERS = { - oxigraph: { - kind: 'local', - requiresExistingPath: false, - }, - 'oxigraph-persistent': { - kind: 'local', - requiresExistingPath: true, - }, - blazegraph: { - kind: 'external', - queryEndpointOption: 'url', - updateEndpointOption: 'url', - }, - 'sparql-http': { - kind: 'external', - queryEndpointOption: 'queryEndpoint', - updateEndpointOption: 'updateEndpoint', - authOption: 'auth', - }, -} as const; - -export type StorageAdapterBackend = keyof typeof STORAGE_ADAPTERS; -export type StorageAdapterPolicy = (typeof STORAGE_ADAPTERS)[StorageAdapterBackend]; -export type StorageAdapterKind = StorageAdapterPolicy['kind']; -export type StorageAdapterOfKind = { - [Backend in StorageAdapterBackend]: typeof STORAGE_ADAPTERS[Backend] extends { kind: Kind } - ? Backend - : never; -}[StorageAdapterBackend]; -export type ExternalStoreBackend = StorageAdapterOfKind<'external'>; -export type LocalStoreBackend = StorageAdapterOfKind<'local'>; - -declare const CUSTOM_TRIPLE_STORE_BACKEND: unique symbol; -export type CustomTripleStoreBackend = string & { - readonly [CUSTOM_TRIPLE_STORE_BACKEND]: true; -}; - -export type ClassifiedTripleStoreBackend = - | { kind: 'adapter'; backend: StorageAdapterBackend } - | { kind: 'custom'; backend: CustomTripleStoreBackend }; - -export function storageAdapterNames(): StorageAdapterBackend[] { - return Object.keys(STORAGE_ADAPTERS) as StorageAdapterBackend[]; -} - -export function isStorageAdapterBackend( - backend: string | undefined | null, -): backend is StorageAdapterBackend { - return backend != null && Object.prototype.hasOwnProperty.call(STORAGE_ADAPTERS, backend); -} - -export function getStorageAdapterPolicy( - backend: string | undefined | null, -): StorageAdapterPolicy | undefined { - return isStorageAdapterBackend(backend) ? STORAGE_ADAPTERS[backend] : undefined; -} - -export function isExternalBackend( - backend: string | undefined | null, -): backend is ExternalStoreBackend { - return isStorageAdapterBackend(backend) && STORAGE_ADAPTERS[backend].kind === 'external'; -} - -export function customTripleStoreBackend(backend: string): CustomTripleStoreBackend { - if (!backend.trim()) throw new Error('Custom triple-store backend name cannot be empty'); - if (isStorageAdapterBackend(backend)) { - throw new Error(`Known triple-store adapter "${backend}" does not need a custom-backend wrapper`); - } - return backend as CustomTripleStoreBackend; -} - -export function classifyTripleStoreBackend(backend: string): ClassifiedTripleStoreBackend { - return isStorageAdapterBackend(backend) - ? { kind: 'adapter', backend } - : { kind: 'custom', backend: backend as CustomTripleStoreBackend }; -} diff --git a/packages/storage/src/triple-store.ts b/packages/storage/src/triple-store.ts index 9ad2a779b7..329c91e529 100644 --- a/packages/storage/src/triple-store.ts +++ b/packages/storage/src/triple-store.ts @@ -17,15 +17,6 @@ import { ChangelogStore, type ChangelogStoreOptions, } from './changelog-store.js'; -import { - classifyTripleStoreBackend, - getStorageAdapterPolicy, - isExternalBackend, - type CustomTripleStoreBackend, - type StorageAdapterBackend, -} from './store-backends.js'; - -export { isExternalBackend } from './store-backends.js'; export interface Quad { subject: string; @@ -175,19 +166,11 @@ export async function tryUpdateWithTouchedGraphs( return true; } -/** - * Backends the storage factory can construct: registered built-in adapters or - * an explicitly branded custom adapter name. - */ -export type TripleStoreBackend = StorageAdapterBackend | CustomTripleStoreBackend; - -/** Explicitly cross from stringly daemon/user config into the factory model. */ -export function toTripleStoreBackend(backend: string): TripleStoreBackend { - const classification = classifyTripleStoreBackend(backend); - return classification.backend; -} +export type TripleStoreBackend = 'oxigraph' | 'oxigraph-persistent' | 'oxigraph-worker' | 'blazegraph' | 'sparql-http' | string; -// The canonical backend taxonomy governs three pieces of daemon behaviour: +// Backends that talk to a remote SPARQL endpoint over HTTP rather than +// owning local files. The local/external split governs three pieces of +// daemon behaviour: // 1. Store-size metric — local backends report file bytes, external // backends have no file to stat (`getStoreBytes` returns null). // 2. Chain-reset wipe — local backends `rm` files, external backends @@ -198,6 +181,12 @@ export function toTripleStoreBackend(backend: string): TripleStoreBackend { // 3. Boot health check — for external backends, an ASK probe runs at // daemon start; an unreachable endpoint exits the daemon with an // actionable message rather than booting half-broken. +const EXTERNAL_BACKENDS: ReadonlySet = new Set(['blazegraph', 'sparql-http']); + +export function isExternalBackend(backend: string | undefined | null): boolean { + return typeof backend === 'string' && EXTERNAL_BACKENDS.has(backend); +} + /** * Shape-normalised SPARQL endpoint extracted from a TripleStoreConfig. * @@ -213,39 +202,34 @@ export interface SparqlEndpoint { headers: Record; } -export interface SparqlEndpointStoreConfig { - backend: string; - options?: Record; -} - -export function getSparqlEndpoint(storeConfig: SparqlEndpointStoreConfig): SparqlEndpoint { +export function getSparqlEndpoint(storeConfig: TripleStoreConfig): SparqlEndpoint { if (!isExternalBackend(storeConfig.backend)) { throw new Error( `getSparqlEndpoint called for non-external backend "${storeConfig.backend}"`, ); } const opts = (storeConfig.options ?? {}) as Record; - const policy = getStorageAdapterPolicy(storeConfig.backend); - if (!policy || policy.kind !== 'external') { - throw new Error(`No external-store policy found for "${storeConfig.backend}"`); + if (storeConfig.backend === 'blazegraph') { + const url = typeof opts.url === 'string' ? opts.url : ''; + if (!url) { + throw new Error('blazegraph storeConfig requires options.url'); + } + return { queryUrl: url, updateUrl: url, headers: {} }; } - const queryOption = policy.queryEndpointOption; - const queryUrl = typeof opts[queryOption] === 'string' ? opts[queryOption] as string : ''; - if (!queryUrl) { - throw new Error(`${storeConfig.backend} storeConfig requires options.${queryOption}`); + // sparql-http + const queryEndpoint = typeof opts.queryEndpoint === 'string' ? opts.queryEndpoint : ''; + if (!queryEndpoint) { + throw new Error('sparql-http storeConfig requires options.queryEndpoint'); } - const updateOption = policy.updateEndpointOption; - const updateUrl = typeof opts[updateOption] === 'string' && opts[updateOption] - ? opts[updateOption] as string - : queryUrl; + const updateEndpoint = + typeof opts.updateEndpoint === 'string' && opts.updateEndpoint + ? opts.updateEndpoint + : queryEndpoint; const headers: Record = {}; - if ('authOption' in policy) { - const auth = opts[policy.authOption]; - if (typeof auth === 'string' && auth) { - headers.Authorization = auth; - } + if (typeof opts.auth === 'string' && opts.auth) { + headers['Authorization'] = opts.auth; } - return { queryUrl, updateUrl, headers }; + return { queryUrl: queryEndpoint, updateUrl: updateEndpoint, headers }; } export interface LargeLiteralStorageConfig { @@ -272,40 +256,22 @@ export interface TripleStoreConfig { changelog?: boolean | ChangelogStoreOptions; } -export type TripleStoreFactoryConfig = TripleStoreConfig; - type AdapterFactory = ( options?: Record, ) => Promise; const adapterRegistry = new Map(); -// Runtime compatibility guard for callers compiled before the worker adapter -// was removed. This is intentionally not part of STORAGE_ADAPTERS: it provides -// a migration error without making retired daemon/config policy constructible. -const REMOVED_ADAPTER_GUIDANCE: Readonly> = { - 'oxigraph-worker': - 'Use "sparql-http" or "blazegraph" for an HTTP store, or ' + - '"oxigraph-persistent" for embedded persistence.', -}; - -export function registerTripleStoreAdapter( - name: Backend, +export function registerTripleStoreAdapter( + name: string, factory: AdapterFactory, -): Backend extends StorageAdapterBackend ? Backend : CustomTripleStoreBackend { +): void { adapterRegistry.set(name, factory); - return name as Backend extends StorageAdapterBackend ? Backend : CustomTripleStoreBackend; } export async function createTripleStore( - config: TripleStoreFactoryConfig, + config: TripleStoreConfig, ): Promise { - const removedGuidance = REMOVED_ADAPTER_GUIDANCE[config.backend]; - if (removedGuidance) { - throw new Error( - `TripleStore backend "${config.backend}" is no longer supported. ${removedGuidance}`, - ); - } const factory = adapterRegistry.get(config.backend); if (!factory) { throw new Error( @@ -371,7 +337,8 @@ function shouldEnableGraphSetIndex(config: TripleStoreConfig): boolean { function isDefaultLocalGraphSetIndexBackend(backend: TripleStoreBackend): boolean { return backend === 'oxigraph' - || backend === 'oxigraph-persistent'; + || backend === 'oxigraph-persistent' + || backend === 'oxigraph-worker'; } function resolveLargeLiteralStorageOptions( diff --git a/packages/storage/test/graph-set-index-store.test.ts b/packages/storage/test/graph-set-index-store.test.ts index 67aaecb55e..751b937fe2 100644 --- a/packages/storage/test/graph-set-index-store.test.ts +++ b/packages/storage/test/graph-set-index-store.test.ts @@ -727,10 +727,8 @@ describe('GraphSetIndexStore', () => { }); it('leaves custom backends uncached unless explicitly enabled', async () => { - const backend = registerTripleStoreAdapter( - 'custom-remote-graph-set-index-test', - async () => new OxigraphStore(), - ); + const backend = 'custom-remote-graph-set-index-test'; + registerTripleStoreAdapter(backend, async () => new OxigraphStore()); const defaultStore = await createTripleStore({ backend }); expect(defaultStore.listGraphsByPrefix).toBeUndefined(); diff --git a/packages/storage/test/is-external-backend.test.ts b/packages/storage/test/is-external-backend.test.ts index daae5378d4..759e08ec35 100644 --- a/packages/storage/test/is-external-backend.test.ts +++ b/packages/storage/test/is-external-backend.test.ts @@ -18,6 +18,7 @@ describe('isExternalBackend', () => { it('returns false for the oxigraph family', () => { expect(isExternalBackend('oxigraph')).toBe(false); + expect(isExternalBackend('oxigraph-worker')).toBe(false); expect(isExternalBackend('oxigraph-persistent')).toBe(false); }); diff --git a/packages/storage/test/oxigraph-worker-resilience.test.ts b/packages/storage/test/oxigraph-worker-resilience.test.ts new file mode 100644 index 0000000000..89c7c6cb98 --- /dev/null +++ b/packages/storage/test/oxigraph-worker-resilience.test.ts @@ -0,0 +1,261 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { OxigraphWorkerStore, createTripleStore, type Quad } from '../src/index.js'; + +// These exercise the embedded worker adapter's resilience guards added to stop +// a single slow/wedged store op from hanging every other store-backed request +// behind it (issues #997 / #999 / #1002 / #1005 / #1008). They need the +// compiled worker artifact (`dist/adapters/oxigraph-worker-impl.js`); if it's +// missing we fail loudly with the remediation hint rather than silently skip, +// matching `storage.test.ts`'s convention. +// +// Design note: the per-op timeout is applied to READ-ONLY ops only. A read is +// side-effect-free, so rejecting it after the bound is a clean, determinate +// failure that surfaces a wedged worker on the exact paths that hang in prod. +// Mutations are left unbounded — timing one out would only drop the caller's +// promise while the write is still in flight, which the rest of the codebase +// would mis-read as a clean failure. So the timeout tests provoke a timeout by +// queuing a READ behind a busy worker, not by bounding a write. +function makeStore(opts?: { operationTimeoutMs?: number }, persistPath?: string): OxigraphWorkerStore { + try { + return new OxigraphWorkerStore(persistPath, opts); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/oxigraph-worker-impl/.test(msg)) { + throw new Error( + `oxigraph-worker adapter is not runnable — run ` + + `\`pnpm --filter @origintrail-official/dkg-storage build\` first. Underlying: ${msg}`, + ); + } + throw err; + } +} + +// For the timeout tests the worker is left mid-operation; close() forcibly +// terminates the thread, but the graceful close reply may itself time out, so +// swallow any error during teardown. +async function closeQuietly(store: OxigraphWorkerStore): Promise { + await store.close().catch(() => {}); +} + +function quads(n: number): Quad[] { + const out: Quad[] = new Array(n); + for (let i = 0; i < n; i += 1) { + out[i] = { + subject: `urn:test:s:${i}`, + predicate: 'http://schema.org/name', + object: `"v${i}"`, + graph: 'urn:test:g', + }; + } + return out; +} + +function abortDuringListenerRegistration(message: string): AbortSignal { + let aborted = false; + let reason: Error | undefined; + return { + get aborted() { + return aborted; + }, + get reason() { + return reason; + }, + addEventListener(type: string, listener: EventListenerOrEventListenerObject) { + if (type !== 'abort') return; + aborted = true; + reason = new Error(message); + if (typeof listener === 'function') listener(new Event('abort')); + else listener.handleEvent(new Event('abort')); + }, + removeEventListener: () => undefined, + dispatchEvent: () => true, + onabort: null, + } as unknown as AbortSignal; +} + +// Occupy the single worker thread with a large UNBOUNDED insert (mutations are +// not bounded by the timeout), so a read posted right after it queues behind a +// busy worker — the production "wedged worker" signature in miniature. +const BUSY_QUERY = 'ASK { GRAPH { ?s ?p ?o } }'; + +describe('OxigraphWorkerStore resilience', () => { + it('rejects a READ queued behind a busy worker after operationTimeoutMs', async () => { + // 50k inserts take hundreds of ms; a 5ms bound on the queued read must + // reject well before the worker frees up. + const store = makeStore({ operationTimeoutMs: 5 }); + try { + const busy = store.insert(quads(50_000)); // occupies the worker (unbounded) + await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); + await busy.catch(() => {}); + } finally { + await closeQuietly(store); + } + }); + + it('surfaces the timeout as a typed OXIGRAPH_WORKER_OP_TIMEOUT error', async () => { + const store = makeStore({ operationTimeoutMs: 5 }); + try { + const busy = store.insert(quads(50_000)); + const err: any = await store.query(BUSY_QUERY).then(() => null, (e) => e); + expect(err).toBeTruthy(); + expect(err.code).toBe('OXIGRAPH_WORKER_OP_TIMEOUT'); + expect(err.method).toBe('query'); + expect(err.timeoutMs).toBe(5); + await busy.catch(() => {}); + } finally { + await closeQuietly(store); + } + }); + + it('rejects a queued READ promptly when the caller aborts', async () => { + const store = makeStore({ operationTimeoutMs: 60_000 }); + try { + const busy = store.insert(quads(50_000)); + const controller = new AbortController(); + const read = store.query(BUSY_QUERY, { signal: controller.signal }); + controller.abort(new Error('caller aborted queued read')); + await expect(read).rejects.toThrow(/caller aborted queued read/); + await busy.catch(() => {}); + } finally { + await closeQuietly(store); + } + }); + + it('rejects when the caller aborts while registering the abort listener', async () => { + const store = makeStore({ operationTimeoutMs: 60_000 }); + try { + await expect( + store.query(BUSY_QUERY, { signal: abortDuringListenerRegistration('listener registration aborted') }), + ).rejects.toThrow(/listener registration aborted/); + } finally { + await closeQuietly(store); + } + }); + + it('does NOT bound mutations — a large insert under a tiny timeout still completes', async () => { + // The whole point of read-only scoping: a write is never reported as a clean + // failure while it's still in flight. With a 5ms bound a 50k insert (>>5ms) + // would reject if it were bounded; it must resolve instead. + const store = makeStore({ operationTimeoutMs: 5 }); + try { + await expect(store.insert(quads(50_000))).resolves.toBeUndefined(); + } finally { + await closeQuietly(store); + } + }); + + it('completes normally within a generous timeout', async () => { + const store = makeStore({ operationTimeoutMs: 60_000 }); + try { + await store.insert(quads(10)); + expect(await store.countQuads('urn:test:g')).toBe(10); + // The data lives in a NAMED graph, so the ASK must scope to it (a bare + // `ASK { ?s ?p ?o }` only matches the default graph). + const r = await store.query(BUSY_QUERY); + expect(r.type).toBe('boolean'); + if (r.type === 'boolean') expect(r.value).toBe(true); + } finally { + await closeQuietly(store); + } + }); + + it('operationTimeoutMs: 0 disables the timeout (a queued read still completes)', async () => { + const store = makeStore({ operationTimeoutMs: 0 }); + try { + const busy = store.insert(quads(50_000)); + // With the bound disabled, the read WAITS for the worker instead of + // rejecting, then returns true once the import lands. + const r = await store.query(BUSY_QUERY); + expect(r.type).toBe('boolean'); + if (r.type === 'boolean') expect(r.value).toBe(true); + await busy; + expect(await store.countQuads('urn:test:g')).toBe(50_000); + } finally { + await closeQuietly(store); + } + }); + + it('a fractional operationTimeoutMs is floored to an integer', async () => { + // normalizeNonNegativeInt floors the ms bound: 5.9 behaves like 5, so the + // surfaced error reports "after 5ms" (no fractional noise). + const store = makeStore({ operationTimeoutMs: 5.9 }); + try { + const busy = store.insert(quads(50_000)); + await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); + await busy.catch(() => {}); + } finally { + await closeQuietly(store); + } + }); + + it('close() is exempt from the per-op timeout so the final flush is never cut short', async () => { + // Codex review: close runs the worker's final flush; bounding it by the + // per-op timeout could terminate() the thread mid-flush and lose writes. + // With a tiny operationTimeoutMs and the worker still busy on a large + // insert, close() must WAIT for the worker to drain rather than reject — + // AND the in-flight write must actually land, not get killed mid-flush. + // We prove durability end-to-end on a persistent path: the regression this + // guards (close-after-debounced-flush race) was a SILENT data loss, so the + // test must assert the quads survive a reopen, not just that close resolves. + const dir = mkdtempSync(join(tmpdir(), 'oxigraph-worker-close-')); + const path = join(dir, 'store.nq'); + try { + const store = makeStore({ operationTimeoutMs: 10 }, path); + const busy = store.insert(quads(50_000)); // occupies the worker + // A read queued behind it times out at 10ms — proves the worker is busy. + await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out/); + // close() must resolve cleanly (not reject with a 10ms timeout): it waits + // for the in-flight op to drain, then flushes + terminates. + await expect(store.close()).resolves.toBeUndefined(); + // The in-flight insert must have COMPLETED (drained), not been terminated + // mid-flight — otherwise close() silently cut it short. + await expect(busy).resolves.toBeUndefined(); + // And the write must be durable: a fresh worker on the same path hydrates + // all 50k quads, proving close() flushed before terminating. + const reopened = makeStore({ operationTimeoutMs: 60_000 }, path); + try { + expect(await reopened.countQuads('urn:test:g')).toBe(50_000); + } finally { + await closeQuietly(reopened); + } + } finally { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } + } + }); + + it('concurrent and repeated close() calls all resolve without hanging', async () => { + // Codex review: close() goes through an UNBOUNDED worker RPC, so a second + // close racing the first was orphaned when terminate() killed the worker and + // never settled. close() is now memoized + the worker 'exit' handler rejects + // anything still pending, so concurrent/repeat closes all settle. + const store = makeStore({ operationTimeoutMs: 60_000 }); + await store.insert(quads(5)); + const results = await Promise.all([store.close(), store.close(), store.close()]); + expect(results).toEqual([undefined, undefined, undefined]); + // Ops issued after close fail fast (store closed) instead of hanging. + await new Promise((r) => setImmediate(r)); + await expect(store.insert(quads(1))).rejects.toThrow(/closed/i); + }); + + it('store.options reach the adapter through createTripleStore (factory path)', async () => { + // Codex review: the user-facing path is createTripleStore({ backend, options }), + // not the constructor — assert the option forwarding in the adapter factory + // actually takes effect so a typo there can't silently drop the knob. + // operationTimeoutMs forwarded: a 5ms bound rejects a read queued behind a + // busy worker. + const store = await createTripleStore({ + backend: 'oxigraph-worker', + options: { operationTimeoutMs: 5 }, + }); + try { + const busy = store.insert(quads(50_000)); + await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); + await busy.catch(() => {}); + } finally { + await store.close().catch(() => {}); + } + }); +}); diff --git a/packages/storage/test/oxigraph-worker-respawn.test.ts b/packages/storage/test/oxigraph-worker-respawn.test.ts new file mode 100644 index 0000000000..6d60206177 --- /dev/null +++ b/packages/storage/test/oxigraph-worker-respawn.test.ts @@ -0,0 +1,348 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { Worker } from 'node:worker_threads'; +import { OxigraphWorkerStore, type Quad } from '../src/index.js'; + +// Regression tests for unexpected-worker-exit recovery. The production +// failure: ERR_WORKER_OUT_OF_MEMORY kills ONLY the worker thread, the daemon +// process keeps running, and the old adapter latched `workerExited` forever — +// a testnet node served HTTP for DAYS with a dead store (green /api/status, +// 503 on every write). The fix auto-respawns the worker on an exit that was +// NOT initiated by close(); these tests kill the REAL worker thread (no +// mocks) by reaching into the private `worker` field and calling terminate(), +// which raises the exact same 'exit'-without-close signal as an OOM kill. +// +// Like the resilience suite next door, this needs the compiled worker +// artifact (`dist/adapters/oxigraph-worker-impl.js`); if it's missing we fail +// loudly with the remediation hint rather than silently skip. +function makeStore(persistPath?: string, opts?: { operationTimeoutMs?: number }): OxigraphWorkerStore { + try { + return new OxigraphWorkerStore(persistPath, opts); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/oxigraph-worker-impl/.test(msg)) { + throw new Error( + `oxigraph-worker adapter is not runnable — run ` + + `\`pnpm --filter @origintrail-official/dkg-storage build\` first. Underlying: ${msg}`, + ); + } + throw err; + } +} + +// The recovery machinery is intentionally private (nothing outside the +// adapter should steer it), so the tests reach in through one typed seam +// instead of scattering `as any` casts around. `lifecycle` is the explicit +// state field the respawn/close/in-memory-loss logic keys off (see the +// WorkerLifecycle model in the adapter); asserting on it pins the state model. +type WorkerLifecycle = + | 'initializing' + | 'live' + | 'respawning' + | 'closing' + | 'closed' + | 'gave_up' + | 'in_memory_lost'; +function internals(store: OxigraphWorkerStore): { + worker: Worker; + lifecycle: WorkerLifecycle; + closePromise: Promise | null; + respawnPromise: Promise | null; + consecutiveRespawnFailures: number; +} { + return store as unknown as { + worker: Worker; + lifecycle: WorkerLifecycle; + closePromise: Promise | null; + respawnPromise: Promise | null; + consecutiveRespawnFailures: number; + }; +} + +/** + * Simulate an OOM-style death of the CURRENT worker thread: terminate() while + * closePromise is null fires the same 'exit' event a crashed thread does. + * Awaiting terminate() guarantees the store's own 'exit' handler already ran + * (it was registered first, and 'exit' listeners run synchronously at emit), + * so on return the respawn has been scheduled — or already completed, for the + * immediate first attempt. + */ +async function killWorker(store: OxigraphWorkerStore): Promise { + await internals(store).worker.terminate(); +} + +function quads(n: number, graph = 'urn:test:g'): Quad[] { + const out: Quad[] = new Array(n); + for (let i = 0; i < n; i += 1) { + out[i] = { + subject: `urn:test:s:${i}`, + predicate: 'http://schema.org/name', + object: `"v${i}"`, + graph, + }; + } + return out; +} + +describe('OxigraphWorkerStore respawn after unexpected worker exit', () => { + let dir: string; + let path: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'oxigraph-worker-respawn-')); + path = join(dir, 'store.nq'); + }); + + afterEach(() => { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } + }); + + it('recovers from an unexpected worker death — subsequent ops succeed on a respawned worker', async () => { + const store = makeStore(path); + try { + await store.insert(quads(10)); + // insert() only schedules the worker's 50ms debounced flush; force the + // data to disk so the respawned worker hydrates it back. + await store.flush(); + + const before = internals(store).worker; + await killWorker(store); + + // Same store OBJECT (every daemon alias keeps working), new thread. + expect(internals(store).worker).not.toBe(before); + + // Reads see the persisted data again — the exact op that used to fail + // forever with "the store is closed". + expect(await store.countQuads('urn:test:g')).toBe(10); + // A disk-persisted store recovers fully: the state model is back to 'live' + // (this is the branch finding 1 deliberately keeps auto-respawning). + expect(internals(store).lifecycle).toBe('live'); + // And writes work too: the store is fully live, not read-only. + await store.insert(quads(5, 'urn:test:g2')); + expect(await store.countQuads('urn:test:g2')).toBe(5); + } finally { + await store.close().catch(() => {}); + } + }); + + it('rejects an in-flight op at kill time with the retryable "restarted — retry" message', async () => { + const store = makeStore(path); + try { + // A 50k insert occupies the single worker thread for hundreds of ms, so + // terminating right after posting is guaranteed to catch it in flight. + const inflight = store.insert(quads(50_000)); + await killWorker(store); + // The outcome of the killed op is genuinely unknown (the thread died + // mid-processing), so the caller gets a RETRYABLE error, not "closed". + await expect(inflight).rejects.toThrow(/oxigraph-worker restarted — retry/); + // ...and a retry against the same store object then succeeds. + await expect(store.insert(quads(10))).resolves.toBeUndefined(); + expect(await store.countQuads('urn:test:g')).toBe(10); + } finally { + await store.close().catch(() => {}); + } + }); + + it('parks ops issued during a respawn backoff instead of failing them', async () => { + const store = makeStore(path); + try { + await store.insert(quads(10)); + await store.flush(); + + // First kill → immediate respawn (backoff tier 0). Kill the replacement + // BEFORE it serves any op, so the second respawn sits in the 1s backoff + // tier — that's the window where new ops must park, not fail. + await killWorker(store); + await killWorker(store); + expect(internals(store).respawnPromise).not.toBeNull(); + + // Issued mid-backoff: the op waits for the replacement worker and then + // succeeds against the disk-persisted state (slightly slower success, + // never a spurious "store is closed"). + expect(await store.countQuads('urn:test:g')).toBe(10); + expect(internals(store).respawnPromise).toBeNull(); + } finally { + await store.close().catch(() => {}); + } + }, 15_000); + + it('a successful op resets the crash-loop counter', async () => { + const store = makeStore(path); + try { + await store.insert(quads(3)); + await store.flush(); + await killWorker(store); + // The respawn consumed one attempt... + expect(internals(store).consecutiveRespawnFailures).toBe(1); + // ...and the first successful reply proves the worker healthy, ending + // the accounting window (occasional crashes days apart never accumulate + // toward the give-up bound). + expect(await store.countQuads('urn:test:g')).toBe(3); + expect(internals(store).consecutiveRespawnFailures).toBe(0); + } finally { + await store.close().catch(() => {}); + } + }); + + it('close() does NOT respawn — the exit stays intentional and ops fail closed', async () => { + const store = makeStore(path); + await store.insert(quads(5)); + const lastWorker = internals(store).worker; + await store.close(); + + // Give any (buggy) respawn scheduling a chance to run before asserting. + await new Promise((r) => setTimeout(r, 50)); + expect(internals(store).respawnPromise).toBeNull(); + expect(internals(store).worker).toBe(lastWorker); + + // Post-close ops fail exactly as before this fix — fast and permanent. + await expect(store.countQuads('urn:test:g')).rejects.toThrow(/store is closed/); + await expect(store.insert(quads(1))).rejects.toThrow(/store is closed/); + }); + + it('gives up after MAX consecutive dead-on-arrival respawns and latches closed with guidance', async () => { + const store = makeStore(path); + try { + await store.insert(quads(2)); + await store.flush(); + // Simulate a genuine crash loop without waiting out the real 1s/5s/30s + // backoff ladder: pre-load the consecutive-failure counter to the bound, + // so the very next unexpected exit hits the give-up branch immediately. + internals(store).consecutiveRespawnFailures = 5; + await killWorker(store); + + // Latched closed — same fail-fast as the pre-recovery behaviour, but the + // error now tells the operator WHY and what to do about it. + const err: any = await store.countQuads('urn:test:g').then(() => null, (e) => e); + expect(err).toBeTruthy(); + expect(err.message).toMatch(/store is closed/); + expect(err.message).toMatch(/respawn gave up/); + // No further respawns get armed by later traffic. + expect(internals(store).respawnPromise).toBeNull(); + // The give-up verdict is now the single, explicit terminal state — no + // cluster of booleans that could disagree with the fail-fast error above. + expect(internals(store).lifecycle).toBe('gave_up'); + } finally { + // close() on a latched store must still resolve (idempotent teardown). + await expect(store.close()).resolves.toBeUndefined(); + } + }); + + it('close() during a respawn backoff wins — the store stays closed', async () => { + const store = makeStore(path); + await store.insert(quads(4)); + await store.flush(); + // Double-kill parks the second respawn in the 1s backoff tier (as above). + await killWorker(store); + await killWorker(store); + expect(internals(store).respawnPromise).not.toBeNull(); + + // An operator close arriving mid-backoff must never be resurrected by the + // supervisor: the pending respawn bails out when it wakes. + await expect(store.close()).resolves.toBeUndefined(); + // Wait out the backoff so a buggy respawn would have fired by now. + await new Promise((r) => setTimeout(r, 1_500)); + await expect(store.countQuads('urn:test:g')).rejects.toThrow(/store is closed/); + }, 15_000); +}); + +// Finding 1 (🔴): an IN-MEMORY store (constructed with NO persistPath) has no +// disk to reload from, so respawning after an unexpected worker exit would come +// up EMPTY yet look perfectly healthy — every row written before the crash +// silently gone (confirmed live: pre-crash data vanished). The fix is to FAIL +// CLOSED on such a store instead of respawning: the worker's data is +// unrecoverable, so subsequent ops must reject with a clear data-loss error, +// never return a wrong-but-plausible empty result. These tests use the REAL +// worker thread (no mocks) exactly like the persistent-store suite above, +// killing it via terminate() to raise the same 'exit'-without-close signal an +// OOM kill would. +describe('OxigraphWorkerStore in-memory fail-closed on unexpected worker exit', () => { + it('fails closed with a data-loss error after an in-memory worker crash — never a silent empty result', async () => { + // No persistPath → in-memory store. This is the exact shape that silently + // lost data before the fix. + const store = makeStore(undefined); + try { + await store.insert(quads(7)); + // Sanity: the data really is there before the crash. + expect(await store.countQuads('urn:test:g')).toBe(7); + + const before = internals(store).worker; + await killWorker(store); + + // The store must NOT respawn (no disk to recover from) — it latches into + // the terminal in_memory_lost state and never arms a replacement worker. + expect(internals(store).lifecycle).toBe('in_memory_lost'); + expect(internals(store).respawnPromise).toBeNull(); + expect(internals(store).worker).toBe(before); // no fresh (empty) thread + + // The regression this guards: a read here used to RESOLVE with 0 (an + // empty respawned store), reporting SUCCESS while all 7 rows were gone. + // `.then(() => null, e => e)` captures a resolve as null and a reject as + // the error, so a truthy `err` proves the read rejected rather than + // silently returning the wrong count. + const err: any = await store.countQuads('urn:test:g').then(() => null, (e) => e); + expect(err).toBeTruthy(); + expect(err.message).toMatch(/IN-MEMORY store's worker crashed/); + expect(err.message).toMatch(/data was lost|cannot be recovered/); + + // Writes fail closed the same way — the store is unusable, not read-only. + await expect(store.insert(quads(1))).rejects.toThrow(/IN-MEMORY store's worker crashed/); + } finally { + // close() on a data-lost store must still resolve (idempotent teardown) + // and must not resurrect it. + await expect(store.close()).resolves.toBeUndefined(); + } + }); + + it('the in_memory_lost state is terminal — later traffic never arms a respawn', async () => { + const store = makeStore(undefined); + try { + await store.insert(quads(3)); + await killWorker(store); + expect(internals(store).lifecycle).toBe('in_memory_lost'); + + // Hammer it with a few more ops: each must fail fast, and none may flip + // the store back to 'respawning'/'live' (the terminal-state guarantee). + for (let i = 0; i < 3; i += 1) { + await expect(store.countQuads('urn:test:g')).rejects.toThrow(/cannot be recovered/); + expect(internals(store).lifecycle).toBe('in_memory_lost'); + expect(internals(store).respawnPromise).toBeNull(); + } + } finally { + await store.close().catch(() => {}); + } + }); + + it('otReviewAgent #1408: the spawn→live transition is guarded — a spawn from a terminal state THROWS (never resurrects the store), but the two legal predecessors enter live', async () => { + // The spawn path (spawnWorker → markSpawnedLive) is the one writer that + // enters 'live'. It must still enforce terminal permanence: were a future + // respawn/close code path to call it after close/give-up/in-memory-loss, it + // has to throw rather than silently reopen the store. Poke the state field + // directly (decoupled from the real worker) to drive the guard. + const store = makeStore(undefined); + const seam = store as unknown as { + lifecycle: WorkerLifecycle; + markSpawnedLive: () => void; + }; + try { + for (const terminal of ['closed', 'gave_up', 'in_memory_lost'] as const) { + seam.lifecycle = terminal; + expect(() => seam.markSpawnedLive()).toThrow(/illegal spawn transition|terminal/i); + expect(seam.lifecycle).toBe(terminal); // stayed terminal — not resurrected + } + // 'closing' is likewise not a legal spawn predecessor. + seam.lifecycle = 'closing'; + expect(() => seam.markSpawnedLive()).toThrow(/illegal spawn transition/i); + // The two legal predecessors DO enter 'live'. + for (const start of ['initializing', 'respawning'] as const) { + seam.lifecycle = start; + expect(() => seam.markSpawnedLive()).not.toThrow(); + expect(seam.lifecycle).toBe('live'); + } + } finally { + await store.close().catch(() => {}); + } + }); +}); diff --git a/packages/storage/test/storage.test.ts b/packages/storage/test/storage.test.ts index 8ac362dca2..d9188f06f2 100644 --- a/packages/storage/test/storage.test.ts +++ b/packages/storage/test/storage.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, expectTypeOf, beforeEach, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeEach, beforeAll, afterAll } from 'vitest'; import { OxigraphStore, BlazegraphStore, @@ -6,8 +6,6 @@ import { GraphManager, PrivateContentStore, createTripleStore, - classifyTripleStoreBackend, - customTripleStoreBackend, loadSelectedSharedMemoryQuads, loadSelectedVerifiableMemoryQuads, registerTripleStoreAdapter, @@ -15,7 +13,6 @@ import { resolveVerifiableMemoryReadGraphs, type Quad, type TripleStore, - type TripleStoreBackend, } from '../src/index.js'; import { contextGraphDataGraphUri, @@ -255,17 +252,12 @@ if (blazeUrl) { // --------------------------------------------------------------------------- describe('createTripleStore factory', () => { - it('keeps managed and retired daemon names out of the constructible backend type', () => { - expectTypeOf<'oxigraph'>().toMatchTypeOf(); - expectTypeOf<'oxigraph-server'>().not.toMatchTypeOf(); - expectTypeOf<'oxigraph-worker'>().not.toMatchTypeOf(); - }); - it('all built-in backends are registered (factory throws something other than "Unknown TripleStore backend")', async () => { // The *registry* contract being tested here is: every built-in // backend name is recognized. The construction itself may require - // options (blazegraph needs `url`, sparql-http needs `queryEndpoint`). - // So a backend passes this test iff calling `createTripleStore` + // options (blazegraph needs `url`, sparql-http needs `queryEndpoint`) + // or worker artifacts (oxigraph-worker needs the compiled worker + // impl). So a backend passes this test iff calling `createTripleStore` // either succeeds OR throws a *non*-"Unknown TripleStore backend" // error. // @@ -274,7 +266,7 @@ describe('createTripleStore factory', () => { // effectively assert "a promise settled" — noise. This version // asserts the positive contract explicitly and points at the // specific failing backend if the registry regresses. - const backends = ['oxigraph', 'blazegraph', 'sparql-http']; + const backends = ['oxigraph', 'oxigraph-worker', 'blazegraph', 'sparql-http']; for (const backend of backends) { let outcome: 'constructed' | Error; try { @@ -320,43 +312,56 @@ describe('createTripleStore factory', () => { ).rejects.toThrow('queryEndpoint'); }); - it('throws on unknown backend', async () => { - await expect(createTripleStore({ backend: customTripleStoreBackend('unknown') })).rejects.toThrow( - 'Unknown TripleStore backend', - ); - }); - - it('gives runtime migration guidance to legacy oxigraph-worker callers', async () => { - await expect(createTripleStore({ - backend: 'oxigraph-worker' as any, - })).rejects.toThrow( - /oxigraph-worker.*no longer supported.*sparql-http.*oxigraph-persistent/, - ); + it('oxigraph-worker adapter is registered and round-trips an insert', async () => { + // The worker adapter resolves `./oxigraph-worker-impl.js` relative to + // the module loaded at runtime. When vitest runs against raw source + // without a prior `pnpm build`, that URL lands in `src/adapters/` where + // only the .ts files live, so the Worker constructor throws + // "Cannot find module … oxigraph-worker-impl.js". + // + // This used to be caught and converted to `ctx.skip()`, which meant a + // green CI run even when the worker artifact was missing — i.e. a + // broken build never triggered a test failure. We now FAIL LOUDLY in + // that case with a remediation hint, so: + // • locally, the developer sees "run pnpm build first" instead of a + // silent skip; + // • in CI, if `pnpm build` was not wired into the lane (or the build + // regresses), this test surfaces it as a red failure. + let store: Awaited>; + try { + store = await createTripleStore({ backend: 'oxigraph-worker' }); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes('Cannot find module') && msg.includes('oxigraph-worker-impl')) { + throw new Error( + `oxigraph-worker adapter is not runnable — the compiled ` + + `oxigraph-worker-impl.js artifact is missing from ` + + `packages/storage/dist/adapters/. Run ` + + `\`pnpm --filter @origintrail-official/dkg-storage build\` ` + + `before running this test. Underlying error: ${msg}`, + ); + } + throw err; + } + await store.insert([{ + subject: 'http://ex.org/s', + predicate: 'http://ex.org/p', + object: '"hi"', + graph: 'http://ex.org/g', + }]); + expect(await store.countQuads()).toBe(1); + await store.close(); }); - it('classifies only factory adapters and leaves daemon policy names outside storage', async () => { - expect(classifyTripleStoreBackend('oxigraph')).toEqual({ - kind: 'adapter', - backend: 'oxigraph', - }); - expect(classifyTripleStoreBackend('oxigraph-server')).toEqual({ - kind: 'custom', - backend: 'oxigraph-server', - }); - expect(classifyTripleStoreBackend('oxigraph-worker')).toEqual({ - kind: 'custom', - backend: 'oxigraph-worker', - }); - await expect(createTripleStore({ - backend: customTripleStoreBackend('oxigraph-server'), - })).rejects.toThrow( - /Unknown TripleStore backend/, + it('throws on unknown backend', async () => { + await expect(createTripleStore({ backend: 'unknown' })).rejects.toThrow( + 'Unknown TripleStore backend', ); }); it('custom adapter can be registered and used', async () => { const calls: string[] = []; - const backend = registerTripleStoreAdapter('test-custom', async () => ({ + registerTripleStoreAdapter('test-custom', async () => ({ insert: async () => { calls.push('insert'); }, delete: async () => {}, deleteByPattern: async () => 0, @@ -370,7 +375,7 @@ describe('createTripleStore factory', () => { close: async () => {}, })); - const store = await createTripleStore({ backend }); + const store = await createTripleStore({ backend: 'test-custom' }); await store.insert([]); expect(calls).toEqual(['insert']); expect(await store.countQuads()).toBe(1); diff --git a/scripts/devnet.sh b/scripts/devnet.sh index 03fabea9a3..ff21784c35 100755 --- a/scripts/devnet.sh +++ b/scripts/devnet.sh @@ -546,7 +546,7 @@ create_node_config() { # and spawns it on its own port — NO Docker. Node 1 is the node the # UI/e2e suite drives, so the suite now exercises the REAL default # backend and deterministically reproduces SPARQL-over-HTTP-only - # bugs such as #996.) + # bugs such as #996 — which the old `oxigraph-worker` default hid.) # Node 3-4: blazegraph (if Docker) else oxigraph (in-process baseline) # Node 5-6: sparql-http → external Dockerized Oxigraph (EXTRA coverage of the # generic external-endpoint path; Docker-only, optional) @@ -573,8 +573,6 @@ create_node_config() { local ox_port_var="OXIGRAPH_SERVER_PORT_${node_num}" local ox_port="${!ox_port_var}" store_block="\"store\": { \"backend\": \"sparql-http\", \"options\": { \"queryEndpoint\": \"http://127.0.0.1:${ox_port}/query\", \"updateEndpoint\": \"http://127.0.0.1:${ox_port}/update\" } }," - else - store_block="\"store\": { \"backend\": \"oxigraph\" }," fi fi @@ -1771,12 +1769,12 @@ cmd_start() { local api_port=$((API_PORT_BASE + i - 1)) local role="edge" [ "$i" -le "$NUM_CORE_NODES" ] && role="core" - local store_label="oxigraph-server" + local store_label="oxigraph-worker" if [ "$i" -ge 3 ] && [ "$i" -le 4 ]; then [ "$BLAZEGRAPH_AVAILABLE" = true ] && store_label="blazegraph" || store_label="oxigraph" fi if [ "$i" -ge 5 ]; then - [ "$OXIGRAPH_SERVER_AVAILABLE" = true ] && store_label="sparql-http" || store_label="oxigraph" + [ "$OXIGRAPH_SERVER_AVAILABLE" = true ] && store_label="oxigraph-server" || store_label="oxigraph-worker" fi log "Node $i ($role, $store_label): http://127.0.0.1:$api_port/ui" done diff --git a/scripts/publisher-smoke-test.sh b/scripts/publisher-smoke-test.sh index 28472b5660..1086fe0cc5 100755 --- a/scripts/publisher-smoke-test.sh +++ b/scripts/publisher-smoke-test.sh @@ -76,7 +76,7 @@ import { DKGPublisher, TripleStoreAsyncLiftPublisher } from '@origintrail-offici const dkgHome = process.env.DKG_HOME; const privateKey = process.env.SMOKE_PRIVATE_KEY; const store = await createTripleStore({ - backend: 'oxigraph-persistent', + backend: 'oxigraph-worker', options: { path: join(dkgHome, 'store.nq') }, }); From ddea0d136118210f4a053b918b65fc0fe4ab6801 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:10:59 +0200 Subject: [PATCH 11/12] fix(agent): bound periodic outbox drains (#1580) (#1623) * fix(agent): bound periodic outbox drains * fix(agent): prevent bounded outbox page starvation * fix(outbox): formalize bounded drain contract * test(devnet): cover bounded outbox scheduling * refactor(outbox): isolate shutdown-safe bounded drainer * test(outbox): assert lexical SQLite tie ordering * test(outbox): close shutdown and paging regressions * test(outbox): pin production paging boundary * fix(outbox): preserve store compatibility --------- Co-authored-by: Branimir Rakic --- packages/agent/src/dkg-agent-lifecycle.ts | 3 +- packages/agent/src/dkg-agent.ts | 8 ++ packages/agent/src/index.ts | 3 +- packages/agent/src/map-with-concurrency.ts | 29 +++++ packages/agent/src/p2p/messenger.ts | 63 ++++++--- packages/agent/src/p2p/outbox-drainer.ts | 93 ++++++++++++++ .../agent/src/sync/catchup-concurrency.ts | 5 + .../agent/src/sync/map-with-concurrency.ts | 57 --------- .../agent/test/map-with-concurrency.test.ts | 3 +- .../agent/test/messenger-substrate.test.ts | 120 +++++++++++++++++- packages/agent/test/outbox-drainer.test.ts | 86 +++++++++++++ .../test/outbox-shutdown-lifecycle.test.ts | 71 +++++++++++ packages/core/src/messenger-types.ts | 20 ++- packages/core/src/protocol-outbox.ts | 38 +++++- packages/core/test/protocol-outbox.test.ts | 78 +++++++++++- packages/node-ui/src/db.ts | 28 +++- .../node-ui/test/messenger-stores.test.ts | 38 ++++++ ...et-test-issue-1580-bounded-outbox-drain.sh | 78 ++++++++++++ 18 files changed, 734 insertions(+), 87 deletions(-) create mode 100644 packages/agent/src/map-with-concurrency.ts create mode 100644 packages/agent/src/p2p/outbox-drainer.ts create mode 100644 packages/agent/src/sync/catchup-concurrency.ts delete mode 100644 packages/agent/src/sync/map-with-concurrency.ts create mode 100644 packages/agent/test/outbox-drainer.test.ts create mode 100644 packages/agent/test/outbox-shutdown-lifecycle.test.ts create mode 100755 scripts/devnet-test-issue-1580-bounded-outbox-drain.sh diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 8a58fafda7..5a74e4693c 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -228,7 +228,8 @@ import { resolveSyncResponderSnapshotBudgetOptions, } from './sync/responder/sync-handler.js'; import { runSyncOnConnect, SyncOnConnectPostSyncError, type SyncOnConnectOutcome, type SyncOnConnectPeerOutcome } from './sync/on-connect/sync-on-connect.js'; -import { mapWithConcurrency, CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/map-with-concurrency.js'; +import { mapWithConcurrency } from './map-with-concurrency.js'; +import { CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/catchup-concurrency.js'; import { getSyncBackpressureSnapshot, getSyncBackpressureBusyError, diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index f27f95994a..0ec0003f03 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -1548,6 +1548,14 @@ export class DKGAgent extends DKGAgentBase { clearInterval(this.messengerOutboxTimer); this.messengerOutboxTimer = null; } + try { + await this.messenger.stopOutboxDrain(); + } catch (error) { + this.log.warn( + createOperationContext('system'), + `DKGAgent.stop: outbox retry drain failed during shutdown: ${error instanceof Error ? error.message : String(error)}`, + ); + } if (this.swmAckQuorumTimer) { clearInterval(this.swmAckQuorumTimer); this.swmAckQuorumTimer = null; diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index f190393ed9..6450cea38e 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -228,7 +228,8 @@ export { // (`/api/context-graph/subscribe` → `catchup-runner-worker-impl`) runs the same // registry-scale per-peer fan-out and must be bounded by the SAME knob, without // deep-importing the compiled `dist/` module. -export { mapWithConcurrency, CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/map-with-concurrency.js'; +export { mapWithConcurrency } from './map-with-concurrency.js'; +export { CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from './sync/catchup-concurrency.js'; // 2026-07-08 sync-storm mitigation (#1233) — resolve the opt-in `agents/_meta` // fetch flag. Exported on the public surface so the CLI daemon lifecycle resolves // it identically to the in-agent lifecycle, without deep-importing `dist/`. diff --git a/packages/agent/src/map-with-concurrency.ts b/packages/agent/src/map-with-concurrency.ts new file mode 100644 index 0000000000..c8a334f690 --- /dev/null +++ b/packages/agent/src/map-with-concurrency.ts @@ -0,0 +1,29 @@ +// Bounded-concurrency ordered map shared by agent subsystems. Keeping this +// neutral avoids coupling p2p retry scheduling to the sync directory. + +/** + * Like `Promise.all(items.map(fn))` but with at most `limit` callbacks in + * flight. Results preserve input order. A rejecting callback rejects the call. + */ +export async function mapWithConcurrency( + items: readonly T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + if (items.length === 0) return []; + if (!Number.isInteger(limit) || limit <= 0 || limit >= items.length) { + return Promise.all(items.map((item, i) => fn(item, i))); + } + + const results = new Array(items.length); + let nextIndex = 0; + const worker = async (): Promise => { + for (;;) { + const i = nextIndex++; + if (i >= items.length) return; + results[i] = await fn(items[i], i); + } + }; + await Promise.all(Array.from({ length: limit }, () => worker())); + return results; +} diff --git a/packages/agent/src/p2p/messenger.ts b/packages/agent/src/p2p/messenger.ts index bfa08b2474..3434af1ded 100644 --- a/packages/agent/src/p2p/messenger.ts +++ b/packages/agent/src/p2p/messenger.ts @@ -11,6 +11,15 @@ import { type ProtocolOutboxEntry, type ProtocolRouter, } from '@origintrail-official/dkg-core'; +import { + OutboxDrainer, + type OutboxDrainerOptions, +} from './outbox-drainer.js'; +export { + DEFAULT_OUTBOX_DRAIN_BATCH_SIZE, + DEFAULT_OUTBOX_DRAIN_CONCURRENCY, + type OutboxDrainerOptions, +} from './outbox-drainer.js'; /** Bytes payload the substrate uses to signal `RESPONSE_GONE` on the wire. */ const RESPONSE_GONE_BYTES = new TextEncoder().encode(RESPONSE_GONE_MARKER); @@ -158,6 +167,11 @@ export interface MessengerDeps { * production uses the default `Date.now`. */ clock?: () => number; + /** + * Periodic retry scheduler bounds. Defaults and validation are owned by + * `OutboxDrainer` (`batchSize: 100`, `concurrency: 4`). + */ + outboxDrain?: OutboxDrainerOptions; } export interface SendOpts { @@ -330,13 +344,13 @@ export interface SloProtocolStats { * via `sloWindowSamples` in `MessengerDeps`. */ export const DEFAULT_SLO_WINDOW_SAMPLES = 1000; - export class Messenger { private readonly router: ProtocolRouter; private readonly idempotencyStore?: MessageIdempotencyStore; private readonly outbox?: ProtocolOutbox; private readonly clock: () => number; private readonly resolvePeer?: (peerId: string, opts: { signal: AbortSignal }) => Promise; + private readonly outboxDrainer?: OutboxDrainer; /** * Application handlers registered via `register`. Stored separately @@ -420,6 +434,13 @@ export class Messenger { this.clock = deps.clock ?? (() => Date.now()); this.sloWindowSamples = deps.sloWindowSamples ?? DEFAULT_SLO_WINDOW_SAMPLES; this.resolvePeer = deps.resolvePeer; + if (this.outbox) { + this.outboxDrainer = new OutboxDrainer( + (now, limit) => this.outbox!.duePage(now, limit), + (entry) => this.retryOutboxEntry(entry), + deps.outboxDrain, + ); + } } /** @@ -823,12 +844,18 @@ export class Messenger { * `dropExpired(now)` evicts it on age — recovering an encoding * bug requires operator action (manual replay or shutdown). */ - async processOutboxTick(now: number): Promise { - if (!this.outbox) return; - const due = this.outbox.due(now); - for (const entry of due) { - await this.retryOutboxEntry(entry); - } + processOutboxTick(now: number): Promise { + return this.outboxDrainer?.tick(now) ?? Promise.resolve(); + } + + /** Await the currently active periodic drain during graceful shutdown. */ + async waitForOutboxDrain(): Promise { + await this.outboxDrainer?.wait(); + } + + /** Cancel the remainder of the loaded page and join already-started retries. */ + async stopOutboxDrain(): Promise { + await this.outboxDrainer?.stop(); } private async retryOutboxEntry(entry: { @@ -867,20 +894,20 @@ export class Messenger { this.clearDhtWalkRateLimitIfDrained(entry.peer); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); + const updated = outbox.enqueueFailure( + entry.peer, + entry.protocol, + entry.messageId, + entry.payload, + errMsg, + this.clock(), + ); if (isRecoverableMessengerSendError(err, errMsg)) { - const updated = outbox.enqueueFailure( - entry.peer, - entry.protocol, - entry.messageId, - entry.payload, - errMsg, - this.clock(), - ); this.maybeScheduleDhtWalk(entry.peer, updated.attempts, errMsg); } - // Non-recoverable: leave the entry alone. `dropExpired` will - // age it out; an operator-facing diagnostic surface (PR-12) - // will surface stuck entries so a human can intervene. + // A non-recoverable retry remains visible for operator intervention, but + // advances on the backoff ladder so it cannot permanently occupy the + // front of every bounded due page and starve later deliverable rows. } finally { outbox.endAttempt(entry.peer, entry.protocol, entry.messageId); } diff --git a/packages/agent/src/p2p/outbox-drainer.ts b/packages/agent/src/p2p/outbox-drainer.ts new file mode 100644 index 0000000000..7c2fba386c --- /dev/null +++ b/packages/agent/src/p2p/outbox-drainer.ts @@ -0,0 +1,93 @@ +import { mapWithConcurrency } from '../map-with-concurrency.js'; + +export const DEFAULT_OUTBOX_DRAIN_BATCH_SIZE = 100; +export const DEFAULT_OUTBOX_DRAIN_CONCURRENCY = 4; + +export interface OutboxDrainerOptions { + batchSize?: number; + concurrency?: number; +} + +interface ResolvedOutboxDrainerOptions { + batchSize: number; + concurrency: number; +} + +function positiveInteger(value: number | undefined, fallback: number, name: string): number { + const resolved = value ?? fallback; + if (!Number.isInteger(resolved) || resolved <= 0) { + throw new RangeError(`OutboxDrainer ${name} must be a positive integer`); + } + return resolved; +} + +/** Shutdown-safe bounded scheduler: its active promise covers every started worker. */ +export class OutboxDrainer { + private active: Promise | null = null; + private stopping = false; + private readonly options: ResolvedOutboxDrainerOptions; + + constructor( + private readonly loadDue: (now: number, limit: number) => readonly T[], + private readonly processEntry: (entry: T) => Promise, + options: OutboxDrainerOptions = {}, + ) { + this.options = { + batchSize: positiveInteger( + options.batchSize, + DEFAULT_OUTBOX_DRAIN_BATCH_SIZE, + 'batchSize', + ), + concurrency: positiveInteger( + options.concurrency, + DEFAULT_OUTBOX_DRAIN_CONCURRENCY, + 'concurrency', + ), + }; + } + + tick(now: number): Promise { + if (this.stopping) return Promise.resolve(); + if (this.active) return this.active; + const drain = this.drain(now); + this.active = drain; + const clearActive = (): void => { + if (this.active === drain) this.active = null; + }; + void drain.then(clearActive, clearActive); + return drain; + } + + async wait(): Promise { + await this.active; + } + + /** Stop admitting work and join retries that had already started. */ + async stop(): Promise { + this.stopping = true; + await this.active; + } + + private async drain(now: number): Promise { + const due = this.loadDue(now, this.options.batchSize).slice(0, this.options.batchSize); + const results = await mapWithConcurrency( + due, + this.options.concurrency, + async (entry): Promise<{ failed: true; error: unknown } | undefined> => { + if (this.stopping) return undefined; + try { + await this.processEntry(entry); + return undefined; + } catch (error) { + return { failed: true, error }; + } + }, + ); + const failures = results + .filter((result): result is { failed: true; error: unknown } => result !== undefined) + .map((result) => result.error); + if (failures.length > 0) { + throw new AggregateError(failures, `${failures.length} outbox retry worker(s) failed`); + } + } +} diff --git a/packages/agent/src/sync/catchup-concurrency.ts b/packages/agent/src/sync/catchup-concurrency.ts new file mode 100644 index 0000000000..83c31c0ba6 --- /dev/null +++ b/packages/agent/src/sync/catchup-concurrency.ts @@ -0,0 +1,5 @@ +/** Sync-owned catch-up policy; generic worker-pool mechanics live separately. */ +export const CATCHUP_MAX_CONCURRENT_PEER_SYNCS: number = (() => { + const raw = Number(process.env.DKG_CATCHUP_MAX_CONCURRENT_PEERS); + return Number.isInteger(raw) && raw > 0 ? raw : 4; +})(); diff --git a/packages/agent/src/sync/map-with-concurrency.ts b/packages/agent/src/sync/map-with-concurrency.ts deleted file mode 100644 index 6f6d834952..0000000000 --- a/packages/agent/src/sync/map-with-concurrency.ts +++ /dev/null @@ -1,57 +0,0 @@ -// map-with-concurrency.ts -// -// Bounded-concurrency ordered map. The catch-up fan-out -// (`runCatchupOverPeers`) previously fired a full durable+SWM sync at EVERY -// connected sync-capable peer via one unbounded `Promise.all` — the top -// amplifier of the 2026-07-07 mainnet "sync storm": one `/api/subscribe` on a -// large-degree node launched N concurrent full-CG pulls, saturating the triple -// store and the muxer. This caps the number of peer syncs running AT ONCE -// without dropping any peer (coverage preserved, just staggered) and without -// changing the result shape callers aggregate over — the returned array is in -// input order, one entry per item, exactly like `Promise.all(items.map(fn))`. - -/** - * Like `Promise.all(items.map(fn))` but with at most `limit` callbacks - * in flight at any moment. Results preserve input order. `fn` receives the - * item and its original index. A rejecting `fn` rejects the whole call (same - * as `Promise.all`) — callers that want per-item isolation must catch inside - * `fn`, as the catch-up fan-out already does. - * - * `limit <= 0` or `limit >= items.length` degrades to a plain `Promise.all` - * (no pool overhead, byte-identical behaviour to the pre-cap code). - */ -export async function mapWithConcurrency( - items: readonly T[], - limit: number, - fn: (item: T, index: number) => Promise, -): Promise { - if (items.length === 0) return []; - if (!Number.isInteger(limit) || limit <= 0 || limit >= items.length) { - return Promise.all(items.map((item, i) => fn(item, i))); - } - - const results = new Array(items.length); - let nextIndex = 0; - const worker = async (): Promise => { - for (;;) { - const i = nextIndex++; - if (i >= items.length) return; - results[i] = await fn(items[i], i); - } - }; - // `limit` workers drain the shared cursor; each awaits its item before - // pulling the next, so no more than `limit` `fn` calls are ever in flight. - await Promise.all(Array.from({ length: limit }, () => worker())); - return results; -} - -/** - * Max peer syncs the catch-up fan-out runs concurrently. Kept small so a - * high-degree node's subscribe/reconcile round can't flood its own triple - * store; every selected peer is still synced, just in bounded waves. - * Env-overridable for operators who need to tune throughput vs. store load. - */ -export const CATCHUP_MAX_CONCURRENT_PEER_SYNCS: number = (() => { - const raw = Number(process.env.DKG_CATCHUP_MAX_CONCURRENT_PEERS); - return Number.isInteger(raw) && raw > 0 ? raw : 4; -})(); diff --git a/packages/agent/test/map-with-concurrency.test.ts b/packages/agent/test/map-with-concurrency.test.ts index 74ad79fb7a..ce27c112ac 100644 --- a/packages/agent/test/map-with-concurrency.test.ts +++ b/packages/agent/test/map-with-concurrency.test.ts @@ -5,7 +5,8 @@ // unchanged), and no more than `limit` callbacks are ever in flight (so a // high-degree node's subscribe round can't flood its own store). import { describe, it, expect } from 'vitest'; -import { mapWithConcurrency, CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from '../src/sync/map-with-concurrency.js'; +import { mapWithConcurrency } from '../src/map-with-concurrency.js'; +import { CATCHUP_MAX_CONCURRENT_PEER_SYNCS } from '../src/sync/catchup-concurrency.js'; const tick = () => new Promise((r) => setTimeout(r, 0)); diff --git a/packages/agent/test/messenger-substrate.test.ts b/packages/agent/test/messenger-substrate.test.ts index eef8324774..5d92d90d97 100644 --- a/packages/agent/test/messenger-substrate.test.ts +++ b/packages/agent/test/messenger-substrate.test.ts @@ -9,7 +9,11 @@ import { type ProtocolRouter, type StreamHandler, } from '@origintrail-official/dkg-core'; -import { Messenger, MessengerNotConfiguredError } from '../src/p2p/messenger.js'; +import { + DEFAULT_OUTBOX_DRAIN_BATCH_SIZE, + Messenger, + MessengerNotConfiguredError, +} from '../src/p2p/messenger.js'; /** * Hand-rolled call recorder: records every call's args on `.calls` @@ -58,7 +62,9 @@ interface RouterDouble { inboundHandler?: StreamHandler; } -function makeRouter(sendImpl?: () => Promise): RouterDouble { +function makeRouter( + sendImpl?: (...args: [string, string, Uint8Array, ...unknown[]]) => Promise, +): RouterDouble { const send = recorder( (sendImpl ?? (async () => new Uint8Array([0x10]))) as ( ...args: [string, string, Uint8Array, ...unknown[]] @@ -411,6 +417,116 @@ describe('Messenger.processOutboxTick (retry loop semantics)', () => { await messenger.processOutboxTick(clock() + 100); expect(router.send.calls.length).toBe(sendCallsBefore); }); + + it('coalesces overlapping ticks and bounds batch size and retry concurrency', async () => { + let active = 0; + let maxActive = 0; + const releases: Array<() => void> = []; + const router = makeRouter(async () => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => releases.push(resolve)); + active -= 1; + return new Uint8Array([0x42]); + }); + const idempotencyStore = new InMemoryMessageIdempotencyStore(); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + for (let i = 0; i < 5; i += 1) { + outboxStore.enqueue(PEER_A, PROTO, `message-${i}`, new Uint8Array([i]), 'offline', 0); + } + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore, + outboxStore, + backoffs: [10], + outboxDrain: { batchSize: 3, concurrency: 2 }, + }); + + const first = messenger.processOutboxTick(100); + const overlapping = messenger.processOutboxTick(100); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(router.send.calls).toHaveLength(2); + expect(maxActive).toBe(2); + releases.splice(0).forEach((release) => release()); + await new Promise((resolve) => setTimeout(resolve, 0)); + releases.splice(0).forEach((release) => release()); + await Promise.all([first, overlapping]); + + expect(router.send.calls).toHaveLength(3); + expect(outboxStore.size()).toBe(2); + }); + + it('caps a production-default tick at the default batch size', async () => { + const router = makeRouter(async () => new Uint8Array([0x42])); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + const queued = DEFAULT_OUTBOX_DRAIN_BATCH_SIZE + 25; + for (let i = 0; i < queued; i += 1) { + outboxStore.enqueue(PEER_A, PROTO, `default-batch-${i}`, new Uint8Array([i]), 'offline', 0); + } + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore: new InMemoryMessageIdempotencyStore(), + outboxStore, + backoffs: [10], + }); + + await messenger.processOutboxTick(100); + + expect(router.send.calls).toHaveLength(DEFAULT_OUTBOX_DRAIN_BATCH_SIZE); + expect(outboxStore.size()).toBe(25); + }); + + it('moves terminal failures behind later due rows instead of starving the next page', async () => { + const router = makeRouter(async (_peer, _protocol, payload) => { + if (payload[0] < 2) throw new Error('Invalid payload'); + return new Uint8Array([0x42]); + }); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + for (let i = 0; i < 3; i += 1) { + outboxStore.enqueue(PEER_A, PROTO, `terminal-${i}`, new Uint8Array([i]), 'offline', 0); + } + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore: new InMemoryMessageIdempotencyStore(), + outboxStore, + backoffs: [10], + clock: () => 100, + outboxDrain: { batchSize: 2, concurrency: 1 }, + }); + + await messenger.processOutboxTick(100); + await messenger.processOutboxTick(100); + expect(router.send.calls).toHaveLength(3); + expect(outboxStore.size()).toBe(2); + }); + + it('waitForOutboxDrain stays pending until the active retry completes', async () => { + let release!: () => void; + const router = makeRouter(async () => { + await new Promise((resolve) => { release = resolve; }); + return new Uint8Array([0x42]); + }); + const outboxStore = new InMemoryProtocolOutboxStore({ backoffs: [10] }); + outboxStore.enqueue(PEER_A, PROTO, FIXED_MSG_ID, new Uint8Array([1]), 'offline', 0); + const messenger = new Messenger({ + router: router as unknown as ProtocolRouter, + idempotencyStore: new InMemoryMessageIdempotencyStore(), + outboxStore, + backoffs: [10], + }); + + const tick = messenger.processOutboxTick(100); + let waitResolved = false; + const waiting = messenger.waitForOutboxDrain().then(() => { waitResolved = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(router.send.calls).toHaveLength(1); + expect(waitResolved).toBe(false); + + release(); + await Promise.all([tick, waiting]); + expect(waitResolved).toBe(true); + expect(outboxStore.size()).toBe(0); + }); }); describe('Messenger construction guardrails', () => { diff --git a/packages/agent/test/outbox-drainer.test.ts b/packages/agent/test/outbox-drainer.test.ts new file mode 100644 index 0000000000..b2a0d6e0a3 --- /dev/null +++ b/packages/agent/test/outbox-drainer.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import { OutboxDrainer } from '../src/p2p/outbox-drainer.js'; + +describe('OutboxDrainer', () => { + it('keeps wait pending until every started worker settles after a sibling failure', async () => { + let release!: () => void; + const blocked = new Promise((resolve) => { release = resolve; }); + const drainer = new OutboxDrainer( + () => ['blocked', 'failed'], + async (entry) => { + if (entry === 'failed') throw new Error('store write failed'); + await blocked; + }, + { batchSize: 2, concurrency: 2 }, + ); + + const tick = drainer.tick(100); + let waitSettled = false; + const waiting = drainer.wait().catch(() => {}).then(() => { waitSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(waitSettled).toBe(false); + release(); + await expect(tick).rejects.toThrow('outbox retry worker'); + await waiting; + expect(waitSettled).toBe(true); + }); + + it('defensively caps a due loader that ignores the requested limit', async () => { + const processed: number[] = []; + const drainer = new OutboxDrainer( + () => [1, 2, 3, 4], + async (entry) => { processed.push(entry); }, + { batchSize: 2, concurrency: 1 }, + ); + + await drainer.tick(100); + expect(processed).toEqual([1, 2]); + }); + + it('starts a fresh drain after a failed tick', async () => { + let fail = true; + let loads = 0; + const drainer = new OutboxDrainer( + () => { loads += 1; return ['entry']; }, + async () => { if (fail) throw new Error('store write failed'); }, + { batchSize: 1, concurrency: 1 }, + ); + + await expect(drainer.tick(100)).rejects.toThrow('outbox retry worker'); + fail = false; + await drainer.tick(200); + expect(loads).toBe(2); + }); + + it('stops pulling new entries while joining retries already in flight', async () => { + let release!: () => void; + const blocked = new Promise((resolve) => { release = resolve; }); + const started: number[] = []; + const drainer = new OutboxDrainer( + () => [1, 2, 3], + async (entry) => { started.push(entry); await blocked; }, + { batchSize: 3, concurrency: 1 }, + ); + + const tick = drainer.tick(100); + await new Promise((resolve) => setTimeout(resolve, 0)); + const stopping = drainer.stop(); + let stopSettled = false; + void stopping.then(() => { stopSettled = true; }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(started).toEqual([1]); + expect(stopSettled).toBe(false); + release(); + await stopping; + expect(stopSettled).toBe(true); + await tick; + expect(started).toEqual([1]); + }); + + it('rejects invalid scheduler bounds at its own boundary', () => { + expect(() => new OutboxDrainer(() => [], async () => {}, { batchSize: 0, concurrency: 1 })) + .toThrow('batchSize must be a positive integer'); + expect(() => new OutboxDrainer(() => [], async () => {}, { batchSize: 1, concurrency: 0 })) + .toThrow('concurrency must be a positive integer'); + }); +}); diff --git a/packages/agent/test/outbox-shutdown-lifecycle.test.ts b/packages/agent/test/outbox-shutdown-lifecycle.test.ts new file mode 100644 index 0000000000..fabc0e4897 --- /dev/null +++ b/packages/agent/test/outbox-shutdown-lifecycle.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from 'vitest'; +import { DKGAgent } from '../src/dkg-agent.js'; + +describe('DKGAgent outbox shutdown lifecycle', () => { + it('cancels and awaits the active outbox drain before network teardown', async () => { + let release!: () => void; + const activeDrain = new Promise((resolve) => { release = resolve; }); + const stopOutboxDrain = vi.fn(() => activeDrain); + const stopNode = vi.fn(async () => {}); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + store: { close: vi.fn(async () => {}) }, + log: { warn: vi.fn() }, + }); + + const stopping = agent.stop(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(stopOutboxDrain).toHaveBeenCalledOnce(); + expect(stopNode).not.toHaveBeenCalled(); + + release(); + await stopping; + expect(stopNode).toHaveBeenCalledOnce(); + }); + + it('logs a failed outbox drain and continues network teardown', async () => { + const stopOutboxDrain = vi.fn(async () => { throw new Error('drain failed'); }); + const stopNode = vi.fn(async () => {}); + const closeStore = vi.fn(async () => {}); + const warn = vi.fn(); + const agent = Object.create(DKGAgent.prototype) as any; + Object.assign(agent, { + started: true, + chainPoller: null, + coreHostRecordingsClosed: false, + drainCoreHostRecordings: vi.fn(async () => {}), + messenger: { stopOutboxDrain }, + clearRandomSamplingBindRetry: vi.fn(), + clearStorageACKRegistrationRetry: vi.fn(), + storageACKRegistrationRetryInFlight: false, + randomSamplingHandle: null, + inFlightSubstrateFanOutCount: () => 0, + router: { closePooling: vi.fn(async () => {}) }, + node: { stop: stopNode }, + store: { close: closeStore }, + log: { warn }, + }); + + await expect(agent.stop()).resolves.toBeUndefined(); + + expect(stopOutboxDrain).toHaveBeenCalledOnce(); + expect(stopNode).toHaveBeenCalledOnce(); + expect(closeStore).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('outbox retry drain failed during shutdown: drain failed'), + ); + }); +}); diff --git a/packages/core/src/messenger-types.ts b/packages/core/src/messenger-types.ts index f292c887ad..b31925fa6c 100644 --- a/packages/core/src/messenger-types.ts +++ b/packages/core/src/messenger-types.ts @@ -209,12 +209,25 @@ export interface ProtocolOutboxStore { hasPendingFor(peer: string): boolean; /** - * All entries whose `nextAttemptAt <= now`. Used by the periodic - * tick to find what's due for retry, regardless of peer - * reachability. + * All entries whose `nextAttemptAt <= now`. + * + * This remains the required public store contract for compatibility with + * existing/custom stores. `ProtocolOutbox` applies canonical ordering when + * it turns this snapshot into a bounded retry page. */ due(now: number): ProtocolOutboxEntry[]; + /** + * Optional storage-level fast path for an ordered bounded retry page. + * + * `ProtocolOutbox` only calls this with a normalized non-negative integer + * limit. Implementations that opt in MUST select the first `limit` rows in + * ascending `nextAttemptAt`, `firstFailureAt`, then + * `(peer, protocol, messageId)` order. Stores that only implement the legacy + * `due(now)` API remain fully supported through the wrapper fallback. + */ + duePage?(now: number, limit: number): ProtocolOutboxEntry[]; + /** * Drop entries whose `firstFailureAt` is older than the * configured max-age. Returns the dropped entries so the caller @@ -244,6 +257,7 @@ export interface ProtocolOutboxStore { getEntry(peer: string, protocol: string, messageId: string): ProtocolOutboxEntry | undefined; } + /** * Durable per-author KA-number allocator (OT-RFC-43 Option-1 * deterministic KA identity, B2 allocator core). diff --git a/packages/core/src/protocol-outbox.ts b/packages/core/src/protocol-outbox.ts index 53e55f5726..c683e240b3 100644 --- a/packages/core/src/protocol-outbox.ts +++ b/packages/core/src/protocol-outbox.ts @@ -89,6 +89,19 @@ function cloneOutboxEntry(entry: ProtocolOutboxEntry): ProtocolOutboxEntry { return { ...entry, payload: cloneBytes(entry.payload) }; } +function compareDueEntries(a: ProtocolOutboxEntry, b: ProtocolOutboxEntry): number { + return a.nextAttemptAt - b.nextAttemptAt + || a.firstFailureAt - b.firstFailureAt + || a.peer.localeCompare(b.peer) + || a.protocol.localeCompare(b.protocol) + || a.messageId.localeCompare(b.messageId); +} + +function normalizeDuePageLimit(limit: number | undefined): number | undefined { + if (limit === undefined || !Number.isFinite(limit)) return undefined; + return Math.max(0, Math.floor(limit)); +} + interface ProtocolOutboxStorePolicy extends ProtocolOutboxOptions { backoffFor: (attempts: number) => number; } @@ -193,9 +206,25 @@ export class ProtocolOutbox { return this.store.hasEntry(peer, protocol, messageId); } - /** Entries whose `nextAttemptAt <= now`. */ + /** All due entries in deterministic retry order. */ due(now: number): ProtocolOutboxEntry[] { - return this.store.due(now); + return this.duePage(now); + } + + /** + * Return a canonical retry page while preserving legacy `due(now)` stores. + * Stores may opt into the bounded fast path; the fallback sorts before it + * caps so an older store cannot bypass either the order or the batch bound. + */ + duePage(now: number, limit?: number): ProtocolOutboxEntry[] { + const normalizedLimit = normalizeDuePageLimit(limit); + if (normalizedLimit === 0) return []; + + const snapshot = normalizedLimit !== undefined && this.store.duePage + ? this.store.duePage(now, normalizedLimit) + : this.store.due(now); + const ordered = [...snapshot].sort(compareDueEntries); + return normalizedLimit === undefined ? ordered : ordered.slice(0, normalizedLimit); } hasPendingFor(peer: string): boolean { @@ -321,9 +350,14 @@ export class InMemoryProtocolOutboxStore implements ProtocolOutboxStore { due(now: number): ProtocolOutboxEntry[] { return Array.from(this.entries.values()) .filter((e) => e.nextAttemptAt <= now) + .sort(compareDueEntries) .map(cloneOutboxEntry); } + duePage(now: number, limit: number): ProtocolOutboxEntry[] { + return this.due(now).slice(0, limit); + } + dropExpired(now: number): ProtocolOutboxEntry[] { const dropped: ProtocolOutboxEntry[] = []; for (const [key, entry] of this.entries) { diff --git a/packages/core/test/protocol-outbox.test.ts b/packages/core/test/protocol-outbox.test.ts index ffad293ba5..a85a28f8db 100644 --- a/packages/core/test/protocol-outbox.test.ts +++ b/packages/core/test/protocol-outbox.test.ts @@ -5,7 +5,11 @@ import { InMemoryProtocolOutboxStore, ProtocolOutbox, } from '../src/protocol-outbox.js'; -import { RESPONSE_CACHE_BYTES } from '../src/messenger-types.js'; +import { + RESPONSE_CACHE_BYTES, + type ProtocolOutboxEntry, + type ProtocolOutboxStore, +} from '../src/messenger-types.js'; const PEER_A = '12D3KooWMilesPlaceholder'; const PEER_B = '12D3KooWLexPlaceholder'; @@ -137,6 +141,78 @@ describe('ProtocolOutbox.due / peer presence', () => { expect(outbox.due(expectedNext)).toHaveLength(1); }); + it('bounds due snapshots in retry-time order', () => { + const { outbox } = fixture(); + outbox.enqueueFailure(PEER_A, PROTO, 'third', PAYLOAD, 'e', 3000); + outbox.enqueueFailure(PEER_A, PROTO, 'first', PAYLOAD, 'e', 1000); + outbox.enqueueFailure(PEER_A, PROTO, 'second', PAYLOAD, 'e', 2000); + const now = 3000 + DEFAULT_PROTOCOL_OUTBOX_BACKOFFS_MS[0]; + expect(outbox.duePage(now, 2).map((entry) => entry.messageId)).toEqual(['first', 'second']); + }); + + it('normalizes limits once and deterministically orders exact timestamp ties', () => { + const { outbox } = fixture(); + outbox.enqueueFailure(PEER_A, PROTO, 'z-last', PAYLOAD, 'e', 1000); + outbox.enqueueFailure(PEER_A, PROTO, 'a-first', PAYLOAD, 'e', 1000); + const now = 1000 + DEFAULT_PROTOCOL_OUTBOX_BACKOFFS_MS[0]; + + expect(outbox.duePage(now, 1.9).map((entry) => entry.messageId)).toEqual(['a-first']); + expect(outbox.duePage(now, Number.NaN).map((entry) => entry.messageId)).toEqual(['a-first', 'z-last']); + }); + + it('keeps legacy due-only stores compatible and sorts before applying the cap', () => { + const backing = new InMemoryProtocolOutboxStore(); + const entry = ( + messageId: string, + firstFailureAt: number, + ): ProtocolOutboxEntry => ({ + peer: PEER_A, + protocol: PROTO, + messageId, + payload: PAYLOAD, + attempts: 1, + firstFailureAt, + lastAttemptAt: firstFailureAt, + nextAttemptAt: 100, + lastError: 'offline', + }); + const newer = entry('a-newer-failure', 20); + const older = entry('z-older-failure', 10); + const legacyStore: ProtocolOutboxStore = { + enqueue: backing.enqueue.bind(backing), + markDelivered: backing.markDelivered.bind(backing), + hasEntry: backing.hasEntry.bind(backing), + hasPendingFor: backing.hasPendingFor.bind(backing), + due: () => [newer, older], + dropExpired: backing.dropExpired.bind(backing), + size: backing.size.bind(backing), + list: backing.list.bind(backing), + getEntry: backing.getEntry.bind(backing), + }; + const outbox = new ProtocolOutbox(legacyStore); + + expect(outbox.duePage(100, 1).map((candidate) => candidate.messageId)) + .toEqual(['z-older-failure']); + expect(outbox.due(100).map((candidate) => candidate.messageId)) + .toEqual(['z-older-failure', 'a-newer-failure']); + }); + + it('uses firstFailureAt before key ordering when nextAttemptAt ties', () => { + const store = new InMemoryProtocolOutboxStore({ backoffs: [50, 10] }); + const outbox = new ProtocolOutbox(store, { backoffs: [50, 10] }); + outbox.enqueueFailure(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'first', 0); + outbox.enqueueFailure(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'second', 90); + outbox.enqueueFailure(PEER_A, PROTO, 'a-newer-failure', PAYLOAD, 'first', 50); + + const due = outbox.duePage(100, 1); + expect(due).toHaveLength(1); + expect(due[0]).toMatchObject({ + messageId: 'z-older-failure', + firstFailureAt: 0, + nextAttemptAt: 100, + }); + }); + it('hasPendingFor tracks peer rows without exposing a reconnect drain snapshot', () => { const { outbox } = fixture(); outbox.enqueueFailure(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2000); diff --git a/packages/node-ui/src/db.ts b/packages/node-ui/src/db.ts index 5eb161754e..eba2e4edc0 100644 --- a/packages/node-ui/src/db.ts +++ b/packages/node-ui/src/db.ts @@ -2781,7 +2781,10 @@ export class SqliteProtocolOutboxStore implements ProtocolOutboxStore { due(now: number): ProtocolOutboxEntry[] { const rows = this.db .prepare( - `SELECT * FROM protocol_outbox WHERE next_attempt_at <= ?`, + `SELECT * FROM protocol_outbox + WHERE next_attempt_at <= ? + ORDER BY next_attempt_at ASC, first_failure_at ASC, + peer_id ASC, protocol ASC, message_id ASC`, ) .all(now) as Array<{ peer_id: string; @@ -2797,6 +2800,29 @@ export class SqliteProtocolOutboxStore implements ProtocolOutboxStore { return rows.map(SqliteProtocolOutboxStore.rowToEntry); } + duePage(now: number, limit: number): ProtocolOutboxEntry[] { + const rows = this.db + .prepare( + `SELECT * FROM protocol_outbox + WHERE next_attempt_at <= ? + ORDER BY next_attempt_at ASC, first_failure_at ASC, + peer_id ASC, protocol ASC, message_id ASC + LIMIT ?`, + ) + .all(now, limit) as Array<{ + peer_id: string; + protocol: string; + message_id: string; + payload: Buffer; + attempts: number; + first_failure_at: number; + last_attempt_at: number; + next_attempt_at: number; + last_error: string | null; + }>; + return rows.map(SqliteProtocolOutboxStore.rowToEntry); + } + dropExpired(now: number): ProtocolOutboxEntry[] { const cutoff = now - this.maxAgeMs; const rows = this.db diff --git a/packages/node-ui/test/messenger-stores.test.ts b/packages/node-ui/test/messenger-stores.test.ts index b88f8bf619..9b8f59feac 100644 --- a/packages/node-ui/test/messenger-stores.test.ts +++ b/packages/node-ui/test/messenger-stores.test.ts @@ -224,6 +224,44 @@ describe('SqliteProtocolOutboxStore', () => { expect(store.due(1_005_000)).toHaveLength(1); }); + it('due applies a stable database-level batch limit', () => { + const store = new SqliteProtocolOutboxStore(db, { backoffFor: () => 5_000 }); + store.enqueue(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 2_000); + store.enqueue(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1_000); + expect(store.duePage(10_000, 1).map((entry) => entry.messageId)).toEqual([MSG_1]); + expect(store.duePage(10_000, 0)).toEqual([]); + }); + + it('due deterministically breaks exact timestamp ties by peer/protocol/message id', () => { + const store = new SqliteProtocolOutboxStore(db, { backoffFor: () => 5_000 }); + store.enqueue(PEER_B, PROTO, MSG_2, PAYLOAD, 'e', 1_000); + store.enqueue(PEER_A, PROTO, MSG_2, PAYLOAD, 'e', 1_000); + store.enqueue(PEER_A, PROTO, MSG_1, PAYLOAD, 'e', 1_000); + + expect(store.duePage(6_000, 3).map((entry) => [entry.peer, entry.messageId])).toEqual([ + [PEER_B, MSG_2], + [PEER_A, MSG_1], + [PEER_A, MSG_2], + ]); + }); + + it('uses firstFailureAt before key ordering when nextAttemptAt ties', () => { + const store = new SqliteProtocolOutboxStore(db, { + backoffFor: (attempts) => attempts === 1 ? 50 : 10, + }); + store.enqueue(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'first', 0); + store.enqueue(PEER_A, PROTO, 'z-older-failure', PAYLOAD, 'second', 90); + store.enqueue(PEER_A, PROTO, 'a-newer-failure', PAYLOAD, 'first', 50); + + const due = store.duePage(100, 1); + expect(due).toHaveLength(1); + expect(due[0]).toMatchObject({ + messageId: 'z-older-failure', + firstFailureAt: 0, + nextAttemptAt: 100, + }); + }); + it('dropExpired removes entries older than maxAgeMs and returns them', () => { const store = new SqliteProtocolOutboxStore(db, { maxAgeMs: 60_000, diff --git a/scripts/devnet-test-issue-1580-bounded-outbox-drain.sh b/scripts/devnet-test-issue-1580-bounded-outbox-drain.sh new file mode 100755 index 0000000000..95fe74e4e1 --- /dev/null +++ b/scripts/devnet-test-issue-1580-bounded-outbox-drain.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Live-devnet regression for #1580. Seed 250 immediately-due rows and assert one +# scheduler pass consumes/reschedules no more than the configured default batch +# of 100. The unfixed drain loads the full due set in one unbounded pass. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEVNET_DIR="${DEVNET_DIR:-$ROOT/.devnet}" +API_PORT_BASE="${API_PORT_BASE:-9201}" +DB="$DEVNET_DIR/node1/node-ui.db" +PREFIX="devnet-issue-1580-$(date +%s)-" + +fail() { echo "[#1580] FAIL: $*" >&2; exit 1; } +cleanup() { + DB="$DB" PREFIX="$PREFIX" node --input-type=module <<'NODE' >/dev/null 2>&1 || true +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB); +db.prepare('DELETE FROM protocol_outbox WHERE message_id LIKE ?').run(`${process.env.PREFIX}%`); +db.close(); +NODE +} +trap cleanup EXIT +[[ -f "$DB" ]] || fail "node1 database missing; start a devnet first" +. "$ROOT/scripts/devnet-lib.sh" + +status2="$(body_of "$(api 2 GET /api/status)")" +peer2="$(field "$status2" peerId)" +[[ -n "$peer2" ]] || fail "node2 peerId unavailable" + +DB="$DB" PEER="$peer2" PREFIX="$PREFIX" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +import { encodeReliableEnvelope, PROTOCOL_MESSAGE } from './packages/core/dist/index.js'; +const db = new Database(process.env.DB); +const insert = db.prepare(`INSERT INTO protocol_outbox + (peer_id, protocol, message_id, payload, attempts, first_failure_at, + last_attempt_at, next_attempt_at, last_error) + VALUES (?, ?, ?, ?, 1, ?, ?, ?, ?)`); +const now = Date.now(); +const seed = db.transaction(() => { + for (let i = 0; i < 250; i += 1) { + const id = `${process.env.PREFIX}${String(i).padStart(3, '0')}`; + const payload = encodeReliableEnvelope({ + messageId: id, version: 1, tsMs: now, + payload: new TextEncoder().encode(`invalid-inner-payload-${i}`), + }); + insert.run(process.env.PEER, PROTOCOL_MESSAGE, id, Buffer.from(payload), now, now, now - 1, 'probe'); + } +}); +seed(); +db.close(); +NODE + +remaining=250 +for _ in $(seq 1 75); do + remaining="$(DB="$DB" PREFIX="$PREFIX" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare('SELECT COUNT(*) AS n FROM protocol_outbox WHERE message_id LIKE ?').get(`${process.env.PREFIX}%`); +process.stdout.write(String(row.n)); db.close(); +NODE +)" + [[ "$remaining" -lt 250 ]] && break + sleep 1 +done +[[ "$remaining" -lt 250 ]] || fail "periodic scheduler did not start within 75s" +sleep 2 + +remaining="$(DB="$DB" PREFIX="$PREFIX" node --input-type=module <<'NODE' +import Database from 'better-sqlite3'; +const db = new Database(process.env.DB, { readonly: true }); +const row = db.prepare('SELECT COUNT(*) AS n FROM protocol_outbox WHERE message_id LIKE ?').get(`${process.env.PREFIX}%`); +process.stdout.write(String(row.n)); db.close(); +NODE +)" +processed=$((250 - remaining)) +[[ "$processed" -le 100 ]] || fail "one drain processed $processed rows (batch limit is 100)" +[[ "$processed" -gt 0 ]] || fail "no rows were processed" +echo "[#1580] PASS: first live drain was bounded ($processed/250 rows)" From 22a2225289464e06a20fac274053a3b0a75eda09 Mon Sep 17 00:00:00 2001 From: Branimir Rakic <33914812+branarakic@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:11:37 +0200 Subject: [PATCH 12/12] fix(publish): isolate named KA shared-memory scope (#1585) (#1631) * fix(publish): isolate named KA shared-memory scope * fix(storage): expose exact named KA SWM reads * fix(publisher): reconcile exact SWM cleanup ownership * fix(swm): enforce exact named knowledge asset scope * fix(storage): preserve checksum casing in exact SWM lookup * fix(storage): isolate named SWM lifecycle scopes * fix(storage): close named lifecycle clear boundary * fix(storage): preserve finalized cleanup contracts * fix(agent): retain legacy queued KA scope fallback --------- Co-authored-by: Branimir Rakic --- packages/agent/src/dkg-agent-publish.ts | 86 +++++++++- .../test/publish-finalized-agent-lane.test.ts | 150 ++++++++++++++++- packages/publisher/src/dkg-publisher.ts | 98 +++++++++-- .../shared-memory-publish-boundary.test.ts | 153 +++++++++++++++++- packages/storage/src/graph-manager.ts | 146 +++++++++++++++-- packages/storage/src/index.ts | 5 + .../test/graph-manager-swm-bound.test.ts | 36 +++++ .../devnet-test-issue-1585-named-ka-swm.sh | 19 +++ 8 files changed, 655 insertions(+), 38 deletions(-) create mode 100755 scripts/devnet-test-issue-1585-named-ka-swm.sh diff --git a/packages/agent/src/dkg-agent-publish.ts b/packages/agent/src/dkg-agent-publish.ts index 1ccb0ece7f..2ca33f5e04 100644 --- a/packages/agent/src/dkg-agent-publish.ts +++ b/packages/agent/src/dkg-agent-publish.ts @@ -101,7 +101,7 @@ import { assertQuadLiteralsMutf8Safe, } from '@origintrail-official/dkg-core'; import { SpanStatusCode } from '@opentelemetry/api'; -import { GraphManager, PrivateContentStore, createTripleStore, loadSelectedSharedMemoryQuads, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, createTripleStore, loadSharedMemoryQuadsForScope, resolveSharedMemoryScopeWriteGraph, type SharedMemoryGraphScope, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig } from '@origintrail-official/dkg-storage'; import { EVMChainAdapter, NoChainAdapter, enrichEvmError, buildKnowledgeAssetUal, type EVMAdapterConfig, type ChainAdapter, type CreateContextGraphParams, type CreateOnChainContextGraphParams, type CreateOnChainContextGraphResult, type TxResult, type V10PublishingConvictionAccountInfo } from '@origintrail-official/dkg-chain'; import { DKGPublisher, PublishHandler, SharedMemoryHandler, UpdateHandler, ChainEventPoller, AccessHandler, AccessClient, @@ -146,6 +146,7 @@ import { type QueryRequest, type QueryResponse, type QueryAccessConfig, type LookupType, } from '@origintrail-official/dkg-query'; import { DKGAgentWallet, type AgentWallet } from './agent-wallet.js'; +import { unpackKnowledgeAssetId } from './ka-identity.js'; import { ProfileManager } from './profile-manager.js'; import { DiscoveryClient, type SkillSearchOptions, type DiscoveredAgent, type DiscoveredOffering } from './discovery.js'; @@ -426,6 +427,28 @@ function normalizeOptionalContextGraphId(value: string | null | undefined): stri return trimmed ? trimmed : undefined; } +function sharedMemoryScopeForFinalizedLifecycle( + authorAddress: string, + packedKaId: bigint | undefined, +): SharedMemoryGraphScope { + if (packedKaId === undefined) return { kind: 'complete-family' }; + const unpacked = unpackKnowledgeAssetId(packedKaId); + const sealedAuthor = ethers.getAddress(authorAddress); + const packedAuthor = BigInt(unpacked.agentAddress); + // Legacy/mock seals may carry only the low 96-bit KA number. Preserve that + // compatibility by binding a zero packed namespace to the sealed author; + // a real nonzero namespace must still match exactly. + if (packedAuthor !== 0n && ethers.getAddress(unpacked.agentAddress) !== sealedAuthor) { + throw new Error( + `Finalized lifecycle KA id ${packedKaId} is not in author ${sealedAuthor}'s namespace`, + ); + } + return { + kind: 'named-lifecycle', + identity: { agentAddress: sealedAuthor, kaNumber: unpacked.kaNumber }, + }; +} + function rejectOversizedRdfLiterals(quads: Quad[] | undefined, label: string): void { if (!quads || quads.length === 0) return; assertQuadLiteralsMutf8Safe(quads, { label }); @@ -3317,18 +3340,20 @@ export class PublishMethods extends DKGAgentBase { contextGraphId: string, selection: 'all' | { rootEntities: string[] }, subGraphName?: string, + scope: SharedMemoryGraphScope = { kind: 'complete-family' }, ): Promise { const swmGraph = contextGraphSharedMemoryUri(contextGraphId, subGraphName); - return loadSelectedSharedMemoryQuads(this.store, swmGraph, selection, { + const options = { querySource: 'agent.resolveLiftWorkspaceSlice', - rootEntitiesErrorMessage: ({ inputCount, hadInput }) => ( + rootEntitiesErrorMessage: ({ inputCount, hadInput }: { inputCount: number; hadInput: boolean }) => ( hadInput ? `_loadSelectedSWMQuads: no valid rootEntities provided ` + `(all ${inputCount} entries failed IRI validation) ` + `for context graph ${contextGraphId}` : `_loadSelectedSWMQuads: no rootEntities supplied for context graph ${contextGraphId}` ), - }); + } as const; + return loadSharedMemoryQuadsForScope(this.store, swmGraph, selection, scope, options); } /** @@ -3805,6 +3830,10 @@ export class PublishMethods extends DKGAgentBase { ); } } + const sharedMemoryScope = sharedMemoryScopeForFinalizedLifecycle( + seal.authorAddress, + seal.reservedKaId ?? packedKaId, + ); const newMerkleHexBare = ethers.hexlify(seal.merkleRoot).slice(2); let result: PublishResult; @@ -3815,6 +3844,7 @@ export class PublishMethods extends DKGAgentBase { [...request.roots], request.subGraphName, ctx, + sharedMemoryScope, ); } catch (err) { this.log.warn( @@ -4264,6 +4294,10 @@ export class PublishMethods extends DKGAgentBase { ); } } + const sharedMemoryScope = sharedMemoryScopeForFinalizedLifecycle( + seal.authorAddress, + seal.reservedKaId ?? packedKaId, + ); const newMerkleHexBare = ethers.hexlify(seal.merkleRoot).slice(2); @@ -4279,6 +4313,7 @@ export class PublishMethods extends DKGAgentBase { contextGraphId, { rootEntities: seal.rootEntities }, opts?.subGraphName, + sharedMemoryScope, ); const updateAttestation = await this._buildPrecomputedUpdateAttestationForSeal( packedKaId, @@ -4311,6 +4346,7 @@ export class PublishMethods extends DKGAgentBase { seal.rootEntities, opts?.subGraphName, opts?.operationCtx ?? createOperationContext('publishFromSWM'), + sharedMemoryScope, ); } catch (err) { this.log.warn( @@ -4397,6 +4433,7 @@ export class PublishMethods extends DKGAgentBase { contextGraphId, { rootEntities: seal.rootEntities }, opts?.subGraphName, + sharedMemoryScope, ); if (sealedSwmQuads.length === 0) { throw new Error( @@ -4412,8 +4449,8 @@ export class PublishMethods extends DKGAgentBase { subGraphName: opts?.subGraphName, publisherNodeIdentityIdOverride: opts?.publisherNodeIdentityIdOverride, publishEpochs: opts?.publishEpochs, - clearSharedMemoryAfter: opts?.clearSharedMemoryAfter, reservedKaId: recoveredReservedKaId, + sharedMemoryScope, // Wired through to the inner publisher.publish() via // publishFromSharedMemory's `precomputedAttestation` option. // Skips the publisher's signing entirely. @@ -4453,6 +4490,17 @@ export class PublishMethods extends DKGAgentBase { } } + // Exact scope owns published-root cleanup. A caller's explicit request to + // clear every remaining share is a separate family-wide destructive action + // that runs only after a confirmed publish/update. + if (result.status === 'confirmed' && opts?.clearSharedMemoryAfter === true) { + await publisher.clearRemainingSharedMemory( + contextGraphId, + opts?.subGraphName, + opts?.operationCtx ?? createOperationContext('publishFromSWM'), + ); + } + // OT-RFC-43 A2 (decision 2) — stamp the VM pointer on the lifecycle URN // whenever the publish/update is confirmed. (For the mint path this is the // first VM pointer; for the update path the DELETE/INSERT above already set @@ -4769,17 +4817,30 @@ export class PublishMethods extends DKGAgentBase { * CG-DID catalog subject is appended so it is in scope for BOTH the author seal * (`_loadSelectedSWMQuads`) and the publisher's reload — which scope identically. * For `selection: 'all'` the selection is returned unchanged (both already read - * the whole SWM graph). + * the whole SWM graph). The generated floor is written into the same explicit + * graph scope as the publish; otherwise an exact named-lifecycle read would + * correctly exclude a floor left in the legacy bucket. */ async _ensureCuratedCatalogInSwm(this: DKGAgent, contextGraphId: string, selection: 'all' | { rootEntities: string[] }, subGraphName: string | undefined, ctx: OperationContext, + scope: SharedMemoryGraphScope = { kind: 'complete-family' }, ): Promise<'all' | { rootEntities: string[] }> { const swmGraph = contextGraphSharedMemoryUri(contextGraphId, subGraphName); + const catalogTargetGraph = await resolveSharedMemoryScopeWriteGraph( + this.store, + swmGraph, + scope, + { source: 'agent.ensureCuratedCatalogInSwm' }, + ); const cgDid = contextGraphDataUri(contextGraphId); - const catalogQuads = buildPublicProjection({ ual: cgDid, accessPolicy: 'private', graph: swmGraph }); + const catalogQuads = buildPublicProjection({ + ual: cgDid, + accessPolicy: 'private', + graph: catalogTargetGraph, + }); await this.store.insert(catalogQuads); this.log.info( ctx, @@ -4844,6 +4905,7 @@ export class PublishMethods extends DKGAgentBase { * publisher then keeps its existing allocate-at-publish behavior. */ reservedKaId?: bigint; + sharedMemoryScope?: SharedMemoryGraphScope; /** * RFC-001 §9.x — pre-computed attestation captured by * `agent.assertion.finalize()`. When the caller has already @@ -4919,7 +4981,13 @@ export class PublishMethods extends DKGAgentBase { ? generatedPrivateCatalogTripleKeys(contextGraphId) : undefined; if (hasGeneratedPrivateCatalog) { - selection = await this._ensureCuratedCatalogInSwm(contextGraphId, selection, options?.subGraphName, ctx); + selection = await this._ensureCuratedCatalogInSwm( + contextGraphId, + selection, + options?.subGraphName, + ctx, + options?.sharedMemoryScope, + ); } // RFC-001 §9.x — selection-based publish bridge. If the caller @@ -4940,6 +5008,7 @@ export class PublishMethods extends DKGAgentBase { contextGraphId, selection, options?.subGraphName, + options?.sharedMemoryScope, ); if (swmQuads.length > 0) { resolvedSeal = await this._buildPrecomputedAttestationForSelection( @@ -5019,6 +5088,7 @@ export class PublishMethods extends DKGAgentBase { precomputedAttestation: resolvedSeal, // OT-RFC-43 A2 — reuse the finalize-stamped packed kaId (no re-allocate). reservedKaId: options?.reservedKaId, + sharedMemoryScope: options?.sharedMemoryScope, encryptInlinePayload, encryptInlineChunked, }); diff --git a/packages/agent/test/publish-finalized-agent-lane.test.ts b/packages/agent/test/publish-finalized-agent-lane.test.ts index 01b2b9428e..227c3ee643 100644 --- a/packages/agent/test/publish-finalized-agent-lane.test.ts +++ b/packages/agent/test/publish-finalized-agent-lane.test.ts @@ -2,9 +2,14 @@ import { describe, expect, it } from 'vitest'; import { buildAssertionSealQuads, contextGraphAssertionUri, + contextGraphDataUri, + contextGraphSharedMemoryUri, + assertionLifecycleUri, contextGraphMetaUri, + createOperationContext, } from '@origintrail-official/dkg-core'; import { OxigraphStore, type Quad } from '@origintrail-official/dkg-storage'; +import { KA_ID_PRED, VM_CURRENT_ASSERTION_PRED } from '@origintrail-official/dkg-publisher'; import { DKGAgent } from '../src/dkg-agent.js'; const CG = 'publish-agent-lane'; @@ -25,6 +30,39 @@ function makeLog() { } describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { + it('injects the curated catalog floor into the exact named lifecycle graph', async () => { + const store = new OxigraphStore(); + const agent = Object.create(DKGAgent.prototype) as any; + agent.store = store; + agent.log = makeLog(); + const swmGraph = contextGraphSharedMemoryUri(CG); + const exactGraph = `${swmGraph}/${AGENT_B}/1`; + const cgDid = contextGraphDataUri(CG); + + const selection = await agent._ensureCuratedCatalogInSwm( + CG, + { rootEntities: [ROOT] }, + undefined, + createOperationContext('test'), + { + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }, + ); + + expect(selection).toEqual({ rootEntities: [ROOT, cgDid] }); + const exact = await store.query( + `SELECT ?p ?o WHERE { GRAPH <${exactGraph}> { <${cgDid}> ?p ?o } }`, + ); + const legacyBucket = await store.query( + `SELECT ?p ?o WHERE { GRAPH <${swmGraph}> { <${cgDid}> ?p ?o } }`, + ); + expect(exact.type).toBe('bindings'); + expect(exact.type === 'bindings' ? exact.bindings : []).toHaveLength(4); + expect(legacyBucket.type).toBe('bindings'); + expect(legacyBucket.type === 'bindings' ? legacyBucket.bindings : []).toHaveLength(0); + }); + it('reads finalized assertions from the explicitly selected non-default agent lane', async () => { const store = new OxigraphStore(); const assertionUri = contextGraphAssertionUri(CG, AGENT_B, NAME); @@ -51,7 +89,16 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { subGraphName?: string; }> = []; const publishCalls: Array<{ contextGraphId: string; selection: any; opts: any }> = []; - const loadCalls: Array<{ contextGraphId: string; selection: any; subGraphName?: string }> = []; + const remainingClearCalls: any[][] = []; + const loadCalls: Array<{ + contextGraphId: string; + selection: any; + subGraphName?: string; + scope?: { kind: 'complete-family' } | { + kind: 'named-lifecycle'; + identity: { agentAddress: string; kaNumber: bigint }; + }; + }> = []; const agent = Object.create(DKGAgent.prototype) as any; agent.store = store; @@ -73,13 +120,18 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { return agentAddress === AGENT_B; }, clearSwmShareComplete: async () => {}, + clearRemainingSharedMemory: async (...args: any[]) => { remainingClearCalls.push(args); }, }; agent._loadSelectedSWMQuads = async ( contextGraphId: string, selection: any, subGraphName?: string, + scope?: { kind: 'complete-family' } | { + kind: 'named-lifecycle'; + identity: { agentAddress: string; kaNumber: bigint }; + }, ) => { - loadCalls.push({ contextGraphId, selection, subGraphName }); + loadCalls.push({ contextGraphId, selection, subGraphName, scope }); return [{ subject: ROOT, predicate: 'http://schema.org/name', @@ -94,7 +146,7 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { ual: 'did:dkg:test/31337/1', merkleRoot: MERKLE, kaManifest: [], - status: 'tentative', + status: 'confirmed', publicQuads: [], }; }; @@ -103,6 +155,7 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { const result = await agent.publishFromFinalizedAssertion(CG, NAME, { agentAddress: AGENT_B, + clearSharedMemoryAfter: true, }); expect(result.assertionUri).toBe(assertionUri); @@ -116,6 +169,10 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { contextGraphId: CG, selection: { rootEntities: [ROOT] }, subGraphName: undefined, + scope: { + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }, }]); expect(publishCalls).toHaveLength(1); expect(publishCalls[0]).toMatchObject({ @@ -124,6 +181,10 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { }); expect(publishCalls[0]?.opts).toMatchObject({ reservedKaId: RESERVED_KA_ID, + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }, precomputedAttestation: { expectedMerkleRoot: MERKLE, authorAddress: AGENT_B, @@ -131,6 +192,87 @@ describe('DKGAgent publishFromFinalizedAssertion agent lane', () => { reservedKaId: RESERVED_KA_ID, }, }); - expect(result.status).toBe('tentative'); + expect(publishCalls[0]?.opts).not.toHaveProperty('clearSharedMemoryAfter'); + expect(remainingClearCalls).toHaveLength(1); + expect(remainingClearCalls[0].slice(0, 2)).toEqual([CG, undefined]); + expect(result.status).toBe('confirmed'); + }); + + it('cleans only the finalized named lifecycle after a confirmed update', async () => { + const store = new OxigraphStore(); + const assertionUri = contextGraphAssertionUri(CG, AGENT_B, NAME); + const metaGraph = contextGraphMetaUri(CG); + const lifecycleUri = assertionLifecycleUri(CG, AGENT_B, NAME); + await store.insert([ + ...buildAssertionSealQuads({ + assertionUri, + metaGraph, + merkleRoot: MERKLE, + authorAddress: AGENT_B, + authorAttestationR: new Uint8Array(32).fill(1), + authorAttestationVS: new Uint8Array(32).fill(2), + authorSchemeVersion: 1, + chainId: 31337n, + kav10Address: AGENT_B, + reservedKaId: RESERVED_KA_ID, + finalizedAtIso: '2026-01-01T00:00:00.000Z', + rootEntities: [ROOT], + }) as Quad[], + { subject: lifecycleUri, predicate: VM_CURRENT_ASSERTION_PRED, object: '"prior"', graph: metaGraph }, + { subject: lifecycleUri, predicate: KA_ID_PRED, object: '"1"', graph: metaGraph }, + ]); + + const cleanupCalls: any[][] = []; + const loadCalls: any[][] = []; + const agent = Object.create(DKGAgent.prototype) as any; + agent.store = store; + agent.chain = {}; + agent.defaultAgentAddress = AGENT_B; + Object.defineProperty(agent, 'peerId', { value: 'peer-update', configurable: true }); + agent.log = makeLog(); + agent.publisher = { + hasSwmShareComplete: async () => true, + clearSwmShareComplete: async () => {}, + clearPublishedSwmRoots: async (...args: any[]) => { cleanupCalls.push(args); }, + }; + agent._loadSelectedSWMQuads = async (...args: any[]) => { + loadCalls.push(args); + return [{ + subject: ROOT, + predicate: 'http://schema.org/name', + object: '"updated"', + graph: '', + }]; + }; + agent._buildPrecomputedUpdateAttestationForSeal = async () => ({ + expectedNewMerkleRoot: MERKLE, + authorAddress: AGENT_B, + signature: { r: new Uint8Array(32), vs: new Uint8Array(32) }, + schemeVersion: 1, + }); + agent.update = async () => ({ + kaId: RESERVED_KA_ID, + ual: 'did:dkg:test/update/1', + merkleRoot: MERKLE, + kaManifest: [], + status: 'confirmed', + publicQuads: [], + }); + + const result = await agent.publishFromFinalizedAssertion(CG, NAME, { agentAddress: AGENT_B }); + + expect(result.status).toBe('confirmed'); + expect(loadCalls).toHaveLength(1); + expect(loadCalls[0].slice(0, 3)).toEqual([CG, { rootEntities: [ROOT] }, undefined]); + expect(loadCalls[0][3]).toEqual({ + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }); + expect(cleanupCalls).toHaveLength(1); + expect(cleanupCalls[0].slice(0, 3)).toEqual([CG, [ROOT], undefined]); + expect(cleanupCalls[0][4]).toEqual({ + kind: 'named-lifecycle', + identity: { agentAddress: AGENT_B, kaNumber: 1n }, + }); }); }); diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index 1f45c5b9ae..fc889ac964 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -1,9 +1,9 @@ -import type { Quad, TripleStore } from '@origintrail-official/dkg-storage'; +import type { Quad, SharedMemoryGraphScope, TripleStore } from '@origintrail-official/dkg-storage'; import type { ChainAdapter, OnChainPublishResult, AddBatchToContextGraphParams } from '@origintrail-official/dkg-chain'; import { enrichEvmError } from '@origintrail-official/dkg-chain'; import type { EventBus, OperationContext } from '@origintrail-official/dkg-core'; import { DKGEvent, Logger, createOperationContext, sha256, encodeWorkspacePublishRequest, encodeEncryptedWorkspacePayload, encryptWorkspacePayload, contextGraphDataUri, contextGraphDataGraphUri, contextGraphMetaUri, contextGraphAssertionUri, contextGraphLayerUri, MemoryLayer, assertionLifecycleUri, contextGraphSubGraphUri, contextGraphSubGraphMetaUri, SYSTEM_CONTEXT_GRAPHS, validateSubGraphName, isSafeIri, assertSafeIri, assertSafeRdfTerm, assertQuadLiteralsMutf8Safe, DKG_GOSSIP_MAX_MESSAGE_BYTES, SwmGossipPayloadTooLargeError, STORAGE_ACK_MAX_STAGING_BYTES, type Ed25519Keypair, buildAuthorAttestationTypedData, buildUpdateAuthorAttestationTypedData, AUTHOR_SCHEME_VERSION_V1, TrustLevel, TRUST_LEVEL_PREDICATE, assertNoUserAuthoredTrustLevelQuads, buildTrustLevelQuads, isTrustLevelQuad, isSwmMerkleExcludedQuad, WORKSPACE_OWNER_PREDICATE, DKG_ENTITY, DKG_ROOT_ENTITY_LEGACY, ENTITY_PRED_ALT, parseAssertionSealQuads, ASSERTION_SEAL_PREDICATES, sharedMemoryReadBothFilter, DKG_ONTOLOGY } from '@origintrail-official/dkg-core'; -import { GraphManager, PrivateContentStore, loadSelectedSharedMemoryQuads } from '@origintrail-official/dkg-storage'; +import { GraphManager, PrivateContentStore, loadSharedMemoryQuadsForScope, loadSelectedSharedMemoryQuads, resolveSharedMemoryScopeGraphs } from '@origintrail-official/dkg-storage'; import { DEFAULT_PUBLISH_EPOCHS, MAX_PUBLISH_EPOCHS, type Publisher, type PublishOptions, type PublishResult, type KAManifestEntry, type PhaseCallback, type V10CoreNodeACK, type V10ACKProviderParams, type V10ACKProviderObject, type LegacyV10ACKProvider } from './publisher.js'; import { skolemizeByEntity } from './auto-partition.js'; import { withKeyedLocks } from './keyed-lock.js'; @@ -1556,9 +1556,19 @@ export class DKGPublisher implements Publisher { * the existing allocate-at-publish behavior. */ reservedKaId?: bigint; + /** Explicit graph-family boundary; named lifecycles exclude bucket and siblings. */ + sharedMemoryScope?: SharedMemoryGraphScope; }, ): Promise { const ctx = options?.operationCtx ?? createOperationContext('publishFromSWM'); + const sharedMemoryScope: SharedMemoryGraphScope = options?.sharedMemoryScope + ?? { kind: 'complete-family' }; + if (sharedMemoryScope.kind === 'named-lifecycle' && options?.clearSharedMemoryAfter === true) { + throw new Error( + 'clearSharedMemoryAfter cannot be combined with a named-lifecycle shared-memory scope; ' + + 'use complete-family scope for an explicit family-wide clear', + ); + } // Guard: VM publishing requires an on-chain registered context graph. // Skip for mock/none chains (unit tests) — only enforce on real chains. @@ -1601,14 +1611,21 @@ export class DKGPublisher implements Publisher { const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, options?.subGraphName); - const quads = await loadSelectedSharedMemoryQuads(this.store, swmGraph, selection, { - quadFilter: (q) => !isSwmMerkleExcludedQuad(q), - rootEntitiesErrorMessage: ({ inputCount, hadInput }) => ( + const loadOptions = { + quadFilter: (q: Quad) => !isSwmMerkleExcludedQuad(q), + rootEntitiesErrorMessage: ({ inputCount, hadInput }: { inputCount: number; hadInput: boolean }) => ( hadInput ? `No valid rootEntities provided (all ${inputCount} entries failed IRI validation)` : `No rootEntities provided for context graph ${contextGraphId}` ), - }); + }; + const quads = await loadSharedMemoryQuadsForScope( + this.store, + swmGraph, + selection, + sharedMemoryScope, + loadOptions, + ); if (quads.length === 0) { throw new Error(`No quads in shared memory for context graph ${contextGraphId} matching selection`); @@ -1849,7 +1866,13 @@ export class DKGPublisher implements Publisher { // clearSharedMemoryAfter controls only whether the REMAINING unpublished triples are also cleared. if (publishResult.status === 'confirmed') { const kaMap = skolemizeByEntity(quads); - await this.clearPublishedSwmRoots(contextGraphId, [...kaMap.keys()], options?.subGraphName, ctx); + await this.clearPublishedSwmRoots( + contextGraphId, + [...kaMap.keys()], + options?.subGraphName, + ctx, + sharedMemoryScope, + ); // If clearSharedMemoryAfter is explicitly true, also clear any remaining unpublished content. // Default is false: unpublished entities stay in SWM for future publishes. if (options?.clearSharedMemoryAfter === true) { @@ -5448,12 +5471,31 @@ export class DKGPublisher implements Publisher { rootEntities: string[], subGraphName: string | undefined, ctx: OperationContext, + scope: SharedMemoryGraphScope = { kind: 'complete-family' }, ): Promise { if (rootEntities.length === 0) return; + const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, subGraphName); + await this.clearPublishedSwmRootsInGraphs( + contextGraphId, + rootEntities, + subGraphName, + ctx, + await resolveSharedMemoryScopeGraphs(this.store, swmGraph, scope), + scope.kind === 'complete-family' ? 'always' : 'when-no-share-remains', + ); + } + + private async clearPublishedSwmRootsInGraphs( + contextGraphId: string, + rootEntities: string[], + subGraphName: string | undefined, + ctx: OperationContext, + swmGraphsForClear: string[], + metadataPolicy: 'always' | 'when-no-share-remains', + ): Promise { const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, subGraphName); const swmMetaGraph = this.graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); const swmOwnershipKey = subGraphName ? `${contextGraphId}\0${subGraphName}` : contextGraphId; - const swmGraphsForClear = await this.swmGraphsUnder(swmGraph); let ownerDeletedTotal = 0; for (const rootEntity of rootEntities) { for (const g of swmGraphsForClear) { @@ -5463,12 +5505,40 @@ export class DKGPublisher implements Publisher { graph: g, subject: rootEntity, predicate: WORKSPACE_OWNER_PREDICATE, }); } - const ownerDeleted = await this.store.deleteByPattern({ - graph: swmMetaGraph, subject: rootEntity, predicate: WORKSPACE_OWNER_PREDICATE, - }); - ownerDeletedTotal += ownerDeleted; - await this.deleteMetaForRoot(swmMetaGraph, rootEntity); - this.sharedMemoryOwnedEntities.get(swmOwnershipKey)?.delete(rootEntity); + } + // A root-keyed owner row can only remain live while some SWM family graph + // still contains that root. Reconcile the complete selected root set with + // one family-wide read rather than one independent scan per root. + const rootsWithRemainingShares = new Set(); + if (metadataPolicy === 'when-no-share-remains') { + const remaining = await loadSelectedSharedMemoryQuads( + this.store, + swmGraph, + { rootEntities }, + { querySource: 'publisher.clearPublishedNamedKnowledgeAssetRoots.reconcileOwnership' }, + ); + for (const quad of remaining) { + for (const rootEntity of rootEntities) { + if ( + quad.subject === rootEntity + || quad.subject.startsWith(`${rootEntity}/.well-known/genid/`) + ) { + rootsWithRemainingShares.add(rootEntity); + } + } + } + } + for (const rootEntity of rootEntities) { + const shouldClearRootMetadata = metadataPolicy === 'always' + || !rootsWithRemainingShares.has(rootEntity); + if (shouldClearRootMetadata) { + const ownerDeleted = await this.store.deleteByPattern({ + graph: swmMetaGraph, subject: rootEntity, predicate: WORKSPACE_OWNER_PREDICATE, + }); + ownerDeletedTotal += ownerDeleted; + await this.deleteMetaForRoot(swmMetaGraph, rootEntity); + this.sharedMemoryOwnedEntities.get(swmOwnershipKey)?.delete(rootEntity); + } } if (ownerDeletedTotal > 0) { this.log.info(ctx, `Cleared ${ownerDeletedTotal} published SWM triple(s) after confirmed publish`); diff --git a/packages/publisher/test/shared-memory-publish-boundary.test.ts b/packages/publisher/test/shared-memory-publish-boundary.test.ts index 880e09715f..5990f6d251 100644 --- a/packages/publisher/test/shared-memory-publish-boundary.test.ts +++ b/packages/publisher/test/shared-memory-publish-boundary.test.ts @@ -4,6 +4,7 @@ import { TRUST_LEVEL_PREDICATE, TrustLevel, TypedEventBus, + createOperationContext, encodeWorkspacePublishRequest, generateEd25519Keypair, DKG_ENTITY, @@ -23,6 +24,8 @@ const CONTEXT_GRAPH_URI = `did:dkg:context-graph:${CONTEXT_GRAPH}`; const SWM_GRAPH = `did:dkg:context-graph:${CONTEXT_GRAPH}/_shared_memory`; const SWM_META_GRAPH = `did:dkg:context-graph:${CONTEXT_GRAPH}/_shared_memory_meta`; const PER_KA_SWM_GRAPH = `${SWM_GRAPH}/0x1111111111111111111111111111111111111111/1`; +const SAME_AUTHOR_SIBLING_SWM_GRAPH = `${SWM_GRAPH}/0x1111111111111111111111111111111111111111/2`; +const FOREIGN_PER_KA_SWM_GRAPH = `${SWM_GRAPH}/0x2222222222222222222222222222222222222222/9`; const ONTOLOGY_GRAPH = 'did:dkg:context-graph:ontology'; const ON_CHAIN_ID_PREDICATE = 'https://dkg.network/ontology#ContextGraphOnChainId'; const WORKSPACE_OWNER_PREDICATE = 'http://dkg.io/ontology/workspaceOwner'; @@ -66,7 +69,7 @@ async function makeRealPublisher(chain = new NoChainAdapter()) { return { publisher, store }; } -async function makePublisher(chain = new NoChainAdapter()) { +async function makePublisher(chain = new NoChainAdapter(), status: PublishResult['status'] = 'tentative') { const { publisher, store } = await makeRealPublisher(chain); const publishResult: PublishResult = { kaId: 1n, @@ -79,7 +82,7 @@ async function makePublisher(chain = new NoChainAdapter()) { privateTripleCount: 0, }, ], - status: 'tentative', + status, publicQuads: [], }; const publishSpy = recorder(async (..._args: Parameters) => publishResult); @@ -206,6 +209,152 @@ describe('publishFromSharedMemory multi-root selection (OT-RFC-44 / Design B: on ]); }); + it('named-KA scope excludes a foreign share with the same subject IRI', async () => { + const { publisher, store, publishSpy } = await makePublisher(); + await store.insert([ + q('urn:test:root:one', 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q('urn:test:root:one', 'http://schema.org/name', '"same-author-sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + q('urn:test:root:one', 'http://schema.org/name', '"foreign"', FOREIGN_PER_KA_SWM_GRAPH), + q('urn:test:root:one', 'http://schema.org/name', '"legacy-bucket"', SWM_GRAPH), + ]); + + await publisher.publishFromSharedMemory( + CONTEXT_GRAPH, + { rootEntities: ['urn:test:root:one'] }, + { + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }, + ); + + expect(publishSpy.calls[0][0].quads).toEqual([ + { subject: 'urn:test:root:one', predicate: 'http://schema.org/name', object: '"local"', graph: '' }, + ]); + }); + + it('confirmed exact cleanup removes stale ownership when the local KA was the last share', async () => { + const { publisher, store } = await makePublisher(new NoChainAdapter(), 'confirmed'); + const root = 'urn:test:root:one'; + await store.insert([ + q(root, 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q(root, WORKSPACE_OWNER_PREDICATE, '"peer-a"', SWM_META_GRAPH), + ]); + + await publisher.publishFromSharedMemory(CONTEXT_GRAPH, { rootEntities: [root] }, { + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }); + + expect(await store.deleteByPattern({ graph: PER_KA_SWM_GRAPH, subject: root })).toBe(0); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: root })).toBe(0); + const owners = await (publisher as any).sharedMemoryOwnersForPromotion( + CONTEXT_GRAPH, undefined, CONTEXT_GRAPH, [root], + ); + expect(owners.size).toBe(0); + }); + + it('confirmed exact cleanup drains only local data and preserves foreign share ownership', async () => { + const { publisher, store } = await makePublisher(new NoChainAdapter(), 'confirmed'); + const root = 'urn:test:root:one'; + await store.insert([ + q(root, 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q(root, 'http://schema.org/name', '"same-author-sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + q(root, 'http://schema.org/name', '"foreign"', FOREIGN_PER_KA_SWM_GRAPH), + q(root, WORKSPACE_OWNER_PREDICATE, '"peer-foreign"', SWM_META_GRAPH), + ]); + + await publisher.publishFromSharedMemory(CONTEXT_GRAPH, { rootEntities: [root] }, { + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }); + + expect(await store.deleteByPattern({ graph: PER_KA_SWM_GRAPH, subject: root })).toBe(0); + expect(await store.deleteByPattern({ graph: SAME_AUTHOR_SIBLING_SWM_GRAPH, subject: root })).toBe(1); + expect(await store.deleteByPattern({ graph: FOREIGN_PER_KA_SWM_GRAPH, subject: root })).toBe(1); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: root })).toBe(1); + }); + + it('batches multi-root exact-cleanup metadata reconciliation into one family read', async () => { + const { publisher, store } = await makePublisher(new NoChainAdapter(), 'confirmed'); + const consumedOnly = 'urn:test:root:consumed-only'; + const stillShared = 'urn:test:root:still-shared'; + await store.insert([ + q(consumedOnly, 'http://schema.org/name', '"local-one"', PER_KA_SWM_GRAPH), + q(stillShared, 'http://schema.org/name', '"local-two"', PER_KA_SWM_GRAPH), + q(`${stillShared}/.well-known/genid/1`, 'http://schema.org/name', '"sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + q(consumedOnly, WORKSPACE_OWNER_PREDICATE, '"peer-a"', SWM_META_GRAPH), + q(stillShared, WORKSPACE_OWNER_PREDICATE, '"peer-a"', SWM_META_GRAPH), + ]); + const originalQuery = store.query.bind(store); + let familyReads = 0; + store.query = async (...args) => { + if (args[1]?.source === 'publisher.clearPublishedNamedKnowledgeAssetRoots.reconcileOwnership') { + familyReads += 1; + } + return originalQuery(...args); + }; + + await publisher.clearPublishedSwmRoots( + CONTEXT_GRAPH, + [consumedOnly, stillShared], + undefined, + createOperationContext('test'), + { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + ); + + expect(familyReads).toBe(1); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: consumedOnly })).toBe(0); + expect(await store.deleteByPattern({ graph: SWM_META_GRAPH, subject: stillShared })).toBe(1); + }); + + it('rejects a family-wide remaining clear for an exact named lifecycle', async () => { + const { publisher, store, publishSpy } = await makePublisher(new NoChainAdapter(), 'confirmed'); + await store.insert([ + q('urn:test:root:one', 'http://schema.org/name', '"local"', PER_KA_SWM_GRAPH), + q('urn:test:root:two', 'http://schema.org/name', '"sibling"', SAME_AUTHOR_SIBLING_SWM_GRAPH), + ]); + + await expect(publisher.publishFromSharedMemory( + CONTEXT_GRAPH, + { rootEntities: ['urn:test:root:one'] }, + { + clearSharedMemoryAfter: true, + sharedMemoryScope: { + kind: 'named-lifecycle', + identity: { + agentAddress: '0x1111111111111111111111111111111111111111', + kaNumber: 1n, + }, + }, + }, + )).rejects.toThrow(/cannot be combined with a named-lifecycle/); + + expect(publishSpy.calls).toHaveLength(0); + expect(await store.deleteByPattern({ graph: PER_KA_SWM_GRAPH })).toBe(1); + expect(await store.deleteByPattern({ graph: SAME_AUTHOR_SIBLING_SWM_GRAPH })).toBe(1); + }); + it('loads selected data root plus generated private-CG catalog root and threads trusted floor', async () => { const { publisher, store, publishSpy } = await makePublisher(privatePolicyChain()); const cgDid = `did:dkg:context-graph:${CONTEXT_GRAPH}`; diff --git a/packages/storage/src/graph-manager.ts b/packages/storage/src/graph-manager.ts index 1e8ae690a6..87a7edbe9b 100644 --- a/packages/storage/src/graph-manager.ts +++ b/packages/storage/src/graph-manager.ts @@ -47,6 +47,17 @@ export interface SwmKaGraphBound { endNumber: bigint; } +/** Exact identity of one named KA lifecycle's shared-memory graph. */ +export interface NamedKnowledgeAssetGraphIdentity { + agentAddress: string; + kaNumber: bigint; +} + +/** Semantic SWM read boundary: either the complete family or one named lifecycle. */ +export type SharedMemoryGraphScope = + | { kind: 'complete-family' } + | { kind: 'named-lifecycle'; identity: NamedKnowledgeAssetGraphIdentity }; + const SWM_CHILD_AGENT_ADDRESS = /^0x[0-9a-fA-F]{40}$/; const SWM_CHILD_KA_NUMBER = /^\d+$/; @@ -146,9 +157,10 @@ async function listGraphsByPrefix( * paths (recompute mismatch → reject/retry, never accept-with-wrong-data). * * This resolver is COMPLETE and therefore safe everywhere, including the - * merkle-defining publish reads and the StorageACK decline lanes. Pruning lives in - * `resolveKaBoundedSharedMemoryReadGraphs`, which is not part of the package's - * public surface — read its contract before reaching for it. + * merkle-defining publish reads and the StorageACK decline lanes. Generic pruning + * lives in `resolveKaBoundedSharedMemoryReadGraphs`, which is not part of the + * package's public surface. The public exact-named-lifecycle API below is a + * separate semantic boundary, not a range-pruning escape hatch. */ export async function resolveSharedMemoryReadGraphs( store: TripleStore, @@ -230,13 +242,22 @@ export async function loadSelectedSharedMemoryQuads( selection: SharedMemoryReadSelection, options: LoadSelectedSharedMemoryQuadsOptions = {}, ): Promise { - return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, undefined); + return loadSharedMemoryQuadsForScope( + store, + bucketGraph, + selection, + { kind: 'complete-family' }, + options, + ); } /** * Load the SWM quad slice pruned to ONE author's per-KA under-graphs (#1549). * - * UNSAFE ON ITS OWN, and deliberately NOT re-exported from `src/index.ts`. The + * UNSAFE as a generic merkle accelerator. It is module-exported for direct + * tests and internal imports, but is not re-exported from the package entrypoint. + * Exact lifecycle callers use the scoped public loader below. Generic merkle + * callers must use the widening wrapper below. The * pruned graph set is a strict subset of the set `loadSelectedSharedMemoryQuads` * reads, and INV-1 — "a root's quads live only under its own KA number" — is * REFUTED under root recurrence, so this read can legitimately miss quads the @@ -252,7 +273,97 @@ export async function loadKaBoundedSharedMemoryQuads( kaGraphBound: SwmKaGraphBound, options: LoadSelectedSharedMemoryQuadsOptions = {}, ): Promise { - return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, kaGraphBound); + return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, { + kind: 'bounded', + bound: kaGraphBound, + }); +} + +/** + * Load shared memory through one explicit semantic scope. + * + * Higher layers do not translate scope into concrete graph policy: complete + * family and exact named-lifecycle dispatch both stay owned by storage. + */ +export async function loadSharedMemoryQuadsForScope( + store: TripleStore, + bucketGraph: string, + selection: SharedMemoryReadSelection, + scope: SharedMemoryGraphScope, + options: LoadSelectedSharedMemoryQuadsOptions = {}, +): Promise { + return loadSharedMemoryQuadsInternal(store, bucketGraph, selection, options, scope); +} + +interface NamedLifecycleGraphResolution { + canonicalGraph: string; + matchingGraphs: string[]; +} + +async function resolveNamedLifecycleGraphPolicy( + store: TripleStore, + bucketGraph: string, + identity: NamedKnowledgeAssetGraphIdentity, + options?: QueryOptions, +): Promise { + assertSafeIri(bucketGraph); + const { agentAddress, kaNumber } = identity; + if (!SWM_CHILD_AGENT_ADDRESS.test(agentAddress) || kaNumber < 0n) { + throw new Error('Named KA graph identity must contain a 20-byte EVM address and non-negative KA number'); + } + const canonicalGraph = `${bucketGraph}/${agentAddress}/${kaNumber.toString()}`; + assertSafeIri(canonicalGraph); + const matchingGraphs = (await listGraphsByPrefix(store, `${bucketGraph}/`, options)) + .filter((graph) => { + const child = parseBoundableSwmChildGraph(bucketGraph, graph); + return child?.agentAddress.toLowerCase() === agentAddress.toLowerCase() + && child.kaNumber === kaNumber; + }); + return { canonicalGraph, matchingGraphs }; +} + +/** Resolve the concrete graph set for an explicit semantic SWM scope. */ +export async function resolveSharedMemoryScopeGraphs( + store: TripleStore, + bucketGraph: string, + scope: SharedMemoryGraphScope, + options?: QueryOptions, +): Promise { + if (scope.kind === 'complete-family') { + return resolveSharedMemoryReadGraphs(store, bucketGraph, options); + } + const { canonicalGraph, matchingGraphs } = await resolveNamedLifecycleGraphPolicy( + store, + bucketGraph, + scope.identity, + options, + ); + // Preserve the writer's checksum casing while matching EVM identity + // case-insensitively. An absent lifecycle still resolves to its canonical + // candidate so the caller gets a safe empty result. + return matchingGraphs.length > 0 + ? matchingGraphs as NonEmptyGraphList + : [canonicalGraph]; +} + +/** Resolve the single graph to WRITE for a semantic scope. */ +export async function resolveSharedMemoryScopeWriteGraph( + store: TripleStore, + bucketGraph: string, + scope: SharedMemoryGraphScope, + options?: QueryOptions, +): Promise { + assertSafeIri(bucketGraph); + if (scope.kind === 'complete-family') return bucketGraph; + const { canonicalGraph, matchingGraphs } = await resolveNamedLifecycleGraphPolicy( + store, + bucketGraph, + scope.identity, + options, + ); + return matchingGraphs.find((graph) => graph === canonicalGraph) + ?? matchingGraphs.slice().sort()[0] + ?? canonicalGraph; } /** Query-source tags for the three read lanes a bounded slice can take. */ @@ -320,7 +431,9 @@ async function loadSharedMemoryQuadsInternal( bucketGraph: string, selection: SharedMemoryReadSelection, options: LoadSelectedSharedMemoryQuadsOptions, - kaGraphBound: SwmKaGraphBound | undefined, + graphScope: + | { kind: 'bounded'; bound: SwmKaGraphBound } + | SharedMemoryGraphScope, ): Promise { let innerGraphPattern: string; if (selection === 'all') { @@ -353,9 +466,22 @@ async function loadSharedMemoryQuadsInternal( } const queryOptions = mergeQueryOptions(options.queryOptions, options.querySource); - const swmGraphs = kaGraphBound - ? await resolveKaBoundedSharedMemoryReadGraphs(store, bucketGraph, kaGraphBound, queryOptions) - : await resolveSharedMemoryReadGraphs(store, bucketGraph, queryOptions); + let swmGraphs: NonEmptyGraphList; + if (graphScope?.kind === 'bounded') { + swmGraphs = await resolveKaBoundedSharedMemoryReadGraphs( + store, + bucketGraph, + graphScope.bound, + queryOptions, + ); + } else { + swmGraphs = await resolveSharedMemoryScopeGraphs( + store, + bucketGraph, + graphScope, + queryOptions, + ); + } const graphValues = swmGraphs.map((g) => `<${g}>`).join(' '); const result = await store.query(`CONSTRUCT { ?s ?p ?o } WHERE { VALUES ?g { ${graphValues} } diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 5e8f4ee97c..9bc3637e82 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -72,14 +72,19 @@ export { ContextGraphManager, GraphManager, loadSelectedSharedMemoryQuads, + loadSharedMemoryQuadsForScope, loadSharedMemorySliceWithKaBoundFallback, + resolveSharedMemoryScopeGraphs, + resolveSharedMemoryScopeWriteGraph, loadSelectedVerifiableMemoryQuads, resolveSharedMemoryReadGraphs, resolveVerifiableMemoryReadGraphs, type LoadSelectedSharedMemoryQuadsOptions, type LoadSelectedVerifiableMemoryQuadsOptions, type NonEmptyGraphList, + type NamedKnowledgeAssetGraphIdentity, type SharedMemoryReadSelection, + type SharedMemoryGraphScope, type SwmKaGraphBound, type SwmSliceSourceTags, } from './graph-manager.js'; diff --git a/packages/storage/test/graph-manager-swm-bound.test.ts b/packages/storage/test/graph-manager-swm-bound.test.ts index 61af795ed3..7e3d512dc7 100644 --- a/packages/storage/test/graph-manager-swm-bound.test.ts +++ b/packages/storage/test/graph-manager-swm-bound.test.ts @@ -2,8 +2,10 @@ import { describe, it, expect } from 'vitest'; import * as storageIndex from '../src/index.js'; import { createTripleStore, + loadSharedMemoryQuadsForScope, loadSelectedSharedMemoryQuads, loadSharedMemorySliceWithKaBoundFallback, + resolveSharedMemoryScopeWriteGraph, resolveSharedMemoryReadGraphs, type Quad, type SwmKaGraphBound, @@ -234,6 +236,36 @@ describe('resolveSharedMemoryReadGraphs — bound only prunes real SWM children }); describe('the generic SWM loader cannot be pruned (bound is not an option)', () => { + it('exact named-KA reads preserve checksum graph casing and exclude the bucket', async () => { + const store = await createTripleStore({ backend: 'oxigraph' }); + const swm = contextGraphSharedMemoryUri('named-exact-casing'); + const root = 'urn:test:named:root'; + const exact = `${swm}/${AUTHOR_A_MIXED}/7`; + const sameAuthorSibling = `${swm}/${AUTHOR_A_MIXED}/8`; + try { + await store.insert([ + { subject: root, predicate: 'urn:p', object: '"bucket"', graph: swm }, + { subject: root, predicate: 'urn:p', object: '"exact"', graph: exact }, + { subject: root, predicate: 'urn:p', object: '"same-author-sibling"', graph: sameAuthorSibling }, + ]); + + const scope = { + kind: 'named-lifecycle', + identity: { agentAddress: AUTHOR_A, kaNumber: 7n }, + } as const; + const quads = await loadSharedMemoryQuadsForScope( + store, + swm, + { rootEntities: [root] }, + scope, + ); + expect(quads.map((quad) => quad.object)).toEqual(['"exact"']); + expect(await resolveSharedMemoryScopeWriteGraph(store, swm, scope)).toBe(exact); + } finally { + await store.close(); + } + }); + // `kaGraphBound` was removed from `LoadSelectedSharedMemoryQuadsOptions`, so the // four production callers — two of them merkle-DEFINING, one the ACK decline lane // — get a compile error if they try to prune. This pins the runtime half: even if @@ -297,6 +329,10 @@ describe('the generic SWM loader cannot be pruned (bound is not an option)', () expect(storageIndex).not.toHaveProperty('resolveKaBoundedSharedMemoryReadGraphs'); // The safe, fallback-owning primitive IS public. expect(typeof storageIndex.loadSharedMemorySliceWithKaBoundFallback).toBe('function'); + // Named publish flows get a scoped API, not a second range-shaped loader. + expect(typeof storageIndex.loadSharedMemoryQuadsForScope).toBe('function'); + expect(typeof storageIndex.resolveSharedMemoryScopeWriteGraph).toBe('function'); + expect(storageIndex).not.toHaveProperty('loadNamedKnowledgeAssetSharedMemoryQuads'); }); }); diff --git a/scripts/devnet-test-issue-1585-named-ka-swm.sh b/scripts/devnet-test-issue-1585-named-ka-swm.sh new file mode 100755 index 0000000000..d01f9b74e9 --- /dev/null +++ b/scripts/devnet-test-issue-1585-named-ka-swm.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Live order-stress regression for #1585. The first suite intentionally leaves +# named-KA SWM lifecycle residue; the subgraph RS suite then publishes on the +# same running devnet. The buggy family-wide named publish bundles/stomps that +# co-resident state and fails the second suite's merkle/cleanup assertions. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +[[ -d "${DEVNET_DIR:-$ROOT/.devnet}/node1" ]] || { + echo "[#1585] FAIL: start a six-node publisher-enabled devnet first" >&2 + exit 1 +} + +echo "[#1585] phase 1/2: create named-KA lifecycle residue" +pnpm test:devnet:ka-lifecycle-cli +echo "[#1585] phase 2/2: run subgraph publish/RS against the same residue" +pnpm test:devnet:pr1385-subgraph-rs +echo "[#1585] PASS: order-stressed named publish preserved co-resident SWM state"