From 9f8ab855be18e4504462cbaad27faa545bea18cc Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Admin Date: Wed, 23 Sep 2026 17:44:44 +0000 Subject: [PATCH 01/11] docs(audit): annotate explicit-network isolation requirements without changing behavior --- frontend/src/app/universe/chain-network.ts | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/frontend/src/app/universe/chain-network.ts b/frontend/src/app/universe/chain-network.ts index 7ba848b5c9..09567731b7 100644 --- a/frontend/src/app/universe/chain-network.ts +++ b/frontend/src/app/universe/chain-network.ts @@ -36,6 +36,42 @@ export function configuredChainNetworks(env: Env): ChainNetworkConfig { return lastParsed; } +/** + * IMPLEMENTATION-HANDOFF [M23-NET] | defect F-M23-02 | coverage C-NET-EXPLICIT + * Preparation only, 2026-09-23. Dependency: M23-BASE; coordinate all callers + * of configuredChainNetworks and chainNetwork before changing their contract. + * Verified at 079dc0d79755bc986bfae3288ffe0e479da3e1f6: malformed JSON or an + * explicit unsupported network is dropped here, then chainNetwork defaults + * to mainnet. chain-network.spec.ts deliberately asserts this for Dogecoin + * signet/testnet3. This violates the requested separation of test and mainnet + * contexts; it is not evidence of a transaction having been sent incorrectly. + * Governing requirement: user brief network isolation (lines 41-49), with + * Bitcoin Signet specified by BIP 325; do not infer Signet support for other + * chains from Bitcoin's selector or from a generic list of network names. + * 1. Preserve missing configuration and omitted-chain mainnet defaults, but + * represent an explicitly invalid map/entry as a typed unavailable result + * with a reason. Do not erase the error into an absent entry or substitute a + * different network. Preserve valid entries independently where safe. + * 2. Validate keys against the actual Explorer chain registry and each + * authority's declared supported networks. Pin that registry contract first; + * do not derive support from CHAIN_NETWORK_VALUES alone or invent new chains. + * 3. Update callers to stop affected API requests, clear prior-context data, + * and render a recoverable configuration error. Keep request/cache/query + * identities chain-and-network bound; retry only after valid configuration. + * 4. Extend chain-network.spec.ts with invalid JSON, invalid explicit network, + * unknown chain, valid object/string, omitted defaults, and configuration + * correction. Add consumer tests proving zero wrong-network requests and no + * stale mainnet data after an invalid test-network selection or reconnect. + * Commands (declared, not executed on SERVER in this preparation): + * cd frontend; npm run test:ci -- src/app/universe/chain-network.spec.ts + * npm run lint; npm run build:universe + * Acceptance: explicit invalid input never becomes a mainnet request; absent + * defaults stay mainnet; valid chain-specific testnet reads survive Bitcoin + * selector changes, refresh and reconnect. Record actual supported-network + * consumer evidence separately from unit tests. No mainnet test transactions. + * Rollback: revert the coordinated parser/caller change together; preserve + * production defaults and network-scoped caches. No database migration here. + */ function parse(raw: unknown): ChainNetworkConfig { if (raw === undefined || raw === null || raw === '') {return {};} let value: unknown = raw; From 036208e47e8920d104da3e7070d98e16a256c216 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Admin Date: Wed, 23 Sep 2026 17:46:42 +0000 Subject: [PATCH 02/11] docs(audit): annotate missing release evidence archive members --- .../workflows/universe-release-artifact.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/universe-release-artifact.yml b/.github/workflows/universe-release-artifact.yml index 07d2e1acef..53717d096f 100644 --- a/.github/workflows/universe-release-artifact.yml +++ b/.github/workflows/universe-release-artifact.yml @@ -120,6 +120,43 @@ jobs: # compare it against the running release and refuse the hard link when it # differs. That check belongs there, where both trees are visible; here # only one of them is. + # IMPLEMENTATION-HANDOFF [M23-PACK] | F-M23-01 | C-RELEASE-ARCHIVE + # Preparation only, 2026-09-23. Prerequisites: M23-BASE and the existing + # qualified acceptance work; this is not permission to bypass that gate. + # Verified at 079dc0d79755bc986bfae3288ffe0e479da3e1f6: this step copies + # two files into stage/docs, but tar below names only backend/frontend/ + # scripts/production/RELEASE-MANIFEST.json. docs is therefore absent. + # release.sh:gate_qualified_acceptance requires the candidate's protocol + # manifest and rooted evidence before cutover. A successful checkout-side + # qualification does not make the extracted archive self-contained. + # Governing sources: scripts/universe/release.sh:gate_qualified_acceptance; + # protocol-contract.mjs:verifyEvidence; user release/evidence requirements. + # 1. Stage the protocol manifest, acceptance envelope and every evidence + # file referenced by that envelope under a candidate-contained root. + # Enumerate and validate the exact reference closure using the existing + # verifier's schema/path rules. Reject missing files, hash mismatches, + # absolute/traversal paths and symlinks escaping the candidate. Do not + # copy credentials, whole workspaces, or unsupported success assertions. + # 2. Include those staged paths in the archive member list. Keep the + # production artifact and evidence bound to the accepted full source SHA + # and production configuration; do not use a mutable external evidence root. + # 3. Extract the actual archive into a new directory, with the checkout + # unavailable, and run the same protocol-contract --release invocation + # with --acceptance-root set to the extracted candidate. Assert all + # referenced evidence members exist with their expected hashes before + # upload. Keep all existing branding, identity and release gates. + # 4. Extend scripts/universe/release-gates.test.mjs with actual tar member + # and extracted-candidate qualification cases, not regex presence alone. + # Cover missing docs, missing nested evidence, tampered bytes, escaped + # symlinks, stale revision and a valid Signet-qualified mainnet envelope. + # Commands declared, not executed on SERVER in this preparation: + # node --test scripts/universe/release-gates.test.mjs + # node --test scripts/universe/protocol-contract.test.mjs + # bash -n scripts/universe/release.sh + # Acceptance: the exact uploaded archive qualifies after extraction with + # no checkout dependency; failed qualification prevents publication/cutover. + # Rollback: retain the previous immutable artifact, routing and compatible + # configuration. No database migration or live transaction is needed here. - name: Pack id: pack run: | From f364046c91facde0dd339ac129b17378c58f412f Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Admin Date: Wed, 23 Sep 2026 17:47:54 +0000 Subject: [PATCH 03/11] docs(audit): annotate capability freshness and operational recovery acceptance --- backend/src/api/capabilities.routes.ts | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/backend/src/api/capabilities.routes.ts b/backend/src/api/capabilities.routes.ts index 54106f8b49..e20c95154f 100644 --- a/backend/src/api/capabilities.routes.ts +++ b/backend/src/api/capabilities.routes.ts @@ -12,6 +12,52 @@ class CapabilitiesRoutes { public initRoutes(app: Application): void { app.get(config.MEMPOOL.API_URL_PREFIX + 'capabilities', async (req: Request, res: Response) => { try { + /* + * IMPLEMENTATION-HANDOFF [M23-HEALTH] | F-M23-03, F-M23-04 + * Coverage: C-BTC-INDEX, C-STATS-FRESH, C-MINING-READY, C-ADDRESS-READY. + * Preparation only, 2026-09-23. Dependencies: M23-BASE, M23-NET. + * Live operational observation, not a mainnet functional test: + * /api/v1/capabilities at 2026-09-23T17:39:41.778Z reported address + * index unavailable, statistics lag 80936 seconds, and mining ready + * despite its indexed height 968172 versus backend-info Core 968299. + * The running backend named 537235052, not the inspected branch tip. + * Verified current-source cause for the readiness gap is in + * capabilities.ts:$miningReport: indexed = total > 0 && poolCount > 0; + * highest/newest are reported but never constrain readiness. Causes + * of the actual address outage and stalled ingestion remain unresolved. + * Sources: Bitcoin Core 31.0 getblockchaininfo RPC; repository + * capabilities.ts, bitcoin/address-index.ts, backend-info-checkpoint.ts; + * user requirements for authoritative freshness and truthful states. + * 1. In capabilities.ts:$miningReport compare the indexed checkpoint + * to a fresh, same-network Core observation and expose the existing + * indexedTip/bitcoinCoreTip/lagBlocks fields. Define a bounded allowed + * lag in the owning configuration contract, tested at its boundary; + * do not infer collector freshness from block mining timestamps alone. + * Preserve unknown separately when a reference checkpoint is absent, + * stale or inconsistent. Existing historical rows must not prove ready. + * 2. Keep this endpoint, release gates and frontend consumers on that + * shared result; coordinate any response/type changes. Exercise cache + * expiry and network identity so a prior ready response cannot conceal + * dependency loss or survive a chain change. Do not substitute HTTP 200 + * or a row count for the final outcome. + * 3. Diagnose the live outage through authorized service/configuration + * identity, cookie freshness, index reachability and ingestion logs. + * Compare Core, completed block cache, SQL tip, address index and + * statistics collector independently. Do not assert one common cause + * without evidence; never restart an active reorg/indexer blindly. + * 4. Add focused tests in the existing capabilities test suite (locate + * before editing): fresh/lagged/unknown tip, database loss, empty rows, + * pool metadata missing, observation expiry, recovery and reorg. + * Run cd backend; npm run test:ci -- --runInBand; npm run lint; + * npm run build. Commands are declared, not executed on SERVER here. + * Acceptance additionally requires real Signet API-to-UI address, + * UTXO/history, mining and statistics outcomes across reload/reconnect; + * faults belong in an isolated environment, not production. Preserve + * historical coverage and accurately report gaps, without fake backfill. + * Rollback: retain the prior artifact, configuration and database backup; + * no destructive rescan, migration or permission relaxation is authorized + * merely by this annotation. Record the verified cause before repair. + */ const report = await capabilities.$report(); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); From 161b7bdd0685631975b95da4f9017db86ba88915 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Admin Date: Wed, 23 Sep 2026 17:52:00 +0000 Subject: [PATCH 04/11] docs(audit): record partial mainnet preparation baseline, findings and remaining gates --- .../mainnet-20260923/README.md | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/implementation-prep/mainnet-20260923/README.md diff --git a/docs/implementation-prep/mainnet-20260923/README.md b/docs/implementation-prep/mainnet-20260923/README.md new file mode 100644 index 0000000000..814f72a904 --- /dev/null +++ b/docs/implementation-prep/mainnet-20260923/README.md @@ -0,0 +1,55 @@ +# Mainnet preparation, 2026-09-23 + +## Decision and evidence limits + +**NO-GO. Preparation is partial; implementation, full functional acceptance and public release are not complete.** No production configuration, runtime logic, permissions, database, live transaction or deployed artifact was changed by this preparation. + +The isolated GitHub branch is `audit/mainnet-prep-20260923-1745`, based on `079dc0d79755bc986bfae3288ffe0e479da3e1f6` (develop). At inspection main was `b2009ac8e2e6a8f594659cc526edd74eea1c534a`; it added a merge commit with no source-tree differences. The source-annotation revision before this index is `f364046c91facde0dd339ac129b17378c58f412f`. Its three commits add 119 comment lines with no deletions. This index is a new non-executable document. + +The SERVER checkout is `D:\universe\mempool\mempool`, whose HEAD file names main. Uncommitted changes, actual Git worktree registration and current service/process state were not established: three Remote Desktop Commander terminal attempts, including read-only Git and Node version commands, were blocked by safety checks. Do not bypass those restrictions or alter the shared checkout. The GitHub branch is real; a prepared SERVER Git worktree is not claimed. + +The created SERVER handoff directory is `D:\universe\mempool\audits\implementation-prep-20260923-1745\mempool_HANDOFF_2026-09-23`. Directory creation alone does not prove a prompt or ZIP was saved; consult the final handoff delivery receipt. Workspace AGENTS.md was read but is not redistributed because it contains credentials. Treat secrets as credentials, never as report content. + +## Source annotation index + +| ID | Actual source anchor | Dependencies | Preparation | Functional status | +|---|---|---|---|---| +| M23-NET | frontend/src/app/universe/chain-network.ts, parse, IMPLEMENTATION-HANDOFF [M23-NET] | M23-BASE | ANNOTATED | FAIL F-M23-02; no repair | +| M23-PACK | .github/workflows/universe-release-artifact.yml, Pack step, IMPLEMENTATION-HANDOFF [M23-PACK] | M23-BASE; existing qualified acceptance work | ANNOTATED | FAIL F-M23-01; no repair | +| M23-HEALTH | backend/src/api/capabilities.routes.ts, capabilities.$report consumer, IMPLEMENTATION-HANDOFF [M23-HEALTH] | M23-BASE, M23-NET | ANNOTATED at API integration point | FAIL F-M23-03/F-M23-04; owning mining helper still needs file-local annotation and implementation | +| M23-BASE | Isolated worktree and running identities | none | BLOCKED on SERVER terminal | NOT TESTED | +| M23-AUTHORITY | backend-apis chain health and named indexers | M23-BASE, M23-NET | NOT ANNOTATED in owning repositories | Operational failures below; individual journeys NOT TESTED | +| M23-COVERAGE | scripts/universe/acceptance-matrix.mjs and owning API registry | M23-BASE | Existing source reviewed; new work not annotated | NOT TESTED; denominator unreconciled | +| M23-ACCEPT | Actual services, UI and protocol operations | all relevant repairs | NOT IMPLEMENTED | NOT TESTED | +| M23-RELEASE | release.sh, accepted artifacts, routing and public endpoints | all acceptance gates | NOT EXECUTED | BLOCKED | + +## Confirmed defects and operational observations + +1. **F-M23-01, release packaging.** The pinned workflow stages `docs/protocols/PROTOCOL-COVERAGE.json` and `docs/acceptance/qualified-release-evidence.json`, then runs `tar -czf "$out" -C "$stage" backend frontend scripts production RELEASE-MANIFEST.json`. Neither document enters the archive. `scripts/universe/release.sh:gate_qualified_acceptance` requires a manifest and candidate-contained evidence before cutover. Repair the member list and complete rooted evidence closure, then qualify the extracted actual artifact without access to the checkout. Missing evidence, wrong hashes, traversal, escaping symlinks and stale candidate identities must fail. Extend `release-gates.test.mjs` beyond string-presence assertions. + +2. **F-M23-02, network isolation.** `parse` drops malformed JSON and unsupported explicit network entries; `chainNetwork` then uses mainnet. The current unit tests intentionally expect this. In the isolated Linux sandbox, actual TypeScript source transpilation reproduced Dogecoin signet and malformed JSON falling back to mainnet. No API request or transaction was sent. Preserve genuinely absent production defaults, but reject explicit invalid contexts through a typed unavailable state, suppress wrong-network requests, clear stale context and update all parser consumers together. + +3. **F-M23-03, mining readiness.** In pinned `backend/src/api/capabilities.ts:$miningReport`, `indexed = total > 0 && poolCount > 0` decides readiness; highest indexed height and data age do not participate. Compare a completed indexed checkpoint to a fresh same-network node checkpoint. Do not equate time since the last mined block with collector health; represent missing/stale reference information separately. + +4. **F-M23-04, Bitcoin operational degradation.** First-party public operational GET observations on 2026-09-23, not mainnet functional tests: + - `/api/v1/backend-info`: release `537235052`, node blocks/headers 968299, initialBlockDownload false, completed explorer checkpoint 968172, a 127-block difference. GitHub resolves that reported prefix to `537235052b9f2d2908aa70c41c4e66c79fcd5e4a` (September 19), not the inspected main revision. + - `/api/v1/capabilities`, generated 17:39:41.778Z: addressLookup unavailable, configured address index did not answer, backendKind electrum. Statistics degraded with 80936 seconds lag. Mining ready despite indexed tip 968172 and 82474 seconds reported age. + - The outage/ingestion root causes are unresolved. Inspect authorized service identity, index reachability, cookie/configuration freshness, Core/cache/SQL checkpoints and collector logs before selecting a repair. Do not blindly restart active indexers or reorg processing. + +5. **F-M23-05, authority availability.** `/api/v1/chains`, observed 17:39:53.152Z, reported ready=false for Bitcoin, Dogecoin and Zcash, with overlay release `fcdc2e2f3bd226bead63cdf0b81fad1ef1ad45bd`. Bitcoin included ten unconfigured authorities and numerous stale/lagged authorities; ready/qualified did not always mean complete historical coverage. Dogecoin block/address history authorities were unavailable; doginals/drc20/dunes were 1380833 blocks behind the node. Zcash zerdinals/zrunes/zrc20 were partial and unqualified. These are operational failures, not proof that each individual transaction journey was executed. Preserve indexing progress, bound load and resolve each authority's own prerequisite. + +6. **F-M23-06, Zcash synchronization inconsistency.** The same response published synced=true and initialBlockDownload=true. Root cause is unresolved. Zcash 6.12.2 RPC documents `initial_block_download_complete`, whose meaning is opposite to Bitcoin's `initialblockdownload`. Inspect the actual deployed node implementation and normalization before changing semantics; do not assume field parity between node families. + +## Coverage and research boundaries + +The pinned protocol roster at `docs/protocols/PROTOCOL-COVERAGE.json` identifies 39 protocols, registry 1.1.0, owner backend-apis revision `7ec4602e7a5dcd6495268ae64698280657cc9c73`. Its seven historical readable declarations are not current test passes. Its read descriptors do not cover the whole application. CAT20/Fractal is the 39th identity outside the three-chain response; absence there is not independently established as a defect. + +The existing `acceptance-matrix.mjs` intentionally records `operationDenominatorReconciled: false` and `FUNCTIONAL NO-GO`. Reconcile its actual source candidates rather than changing those fields to claim completion. Retain all offered APIs, public UI, admin authorization, workers, node/indexer paths, portfolio and observatory/tool capabilities. Do not invent minting, trading or key custody merely because a protocol supports them; do not exclude an actually offered integration because an older README lacks it. + +Primary sources reviewed include Bitcoin Core 31.0 getblockchaininfo RPC, BIP 325 Signet, Zcash 6.12.2 getblockchaininfo RPC, Esplora API and Ord API documentation, plus pinned repository contracts. Complete protocol-by-protocol governing specification/version research and operation traceability were not achieved. Missing pins, exact authority prerequisites and unexecuted checks remain explicit work, not completed research. + +## Validation and next sequence + +Exact UTF-8 snapshots of all three baseline and annotated files were checked against six Git blob hashes. TypeScript 5.8.3 transpilation produced identical comment-stripped JavaScript before/after. Parsed workflow YAML structures were equal. An isolated tar-member reproduction confirmed missing docs. The preparation patch applied to disposable copies and reproduced the annotated bytes. Environment: Linux sandbox, Node v22.16.0, not the SERVER-pinned Node 24.19.0. These results prove preparation/reproduction properties only, not a full build, lint, project typecheck, Signet pass or release. + +Continue M23-BASE, network isolation and authoritative state repair, then per-authority/protocol requirements, persistence/recovery, API/UI integration, complete operation reconciliation and real acceptance, then packaging and gated release. Preserve existing work and first-party-only data infrastructure. Full Signet PASS is functional acceptance where supported; use an explicitly justified supported testnet otherwise. No mainnet funds or test transactions. Test-network/mainnet differences need offline/configuration evidence. Public release follows only after every applicable operation passes and every repair/dependency regression is evidenced. No release is claimed here. From 6aaf932ff58d4635091969611215a8961d2a7a41 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 20:57:02 +0000 Subject: [PATCH 05/11] fix(network): refuse explicit invalid chain network settings instead of reading mainnet M23-NET, F-M23-02. A malformed UNIVERSE_CHAIN_NETWORKS value or an unsupported network used to be dropped with a warning, and the chain then read mainnet. An explicit setting that is wrong now resolves to a typed unavailable result with a reason; absent or omitted settings still read mainnet. Keys and networks are pinned to the overlay's explorer-context contract (dogecoin, zcash, fractal). Every caller changes together: chain reads are deferred and fail with ChainNetworkUnavailableError before any request, the picker skips the chain and names the setting invalid, the websocket opens no socket, bookmarks and visits are never filed under a substitute network, and the dashboard, mining and chain pages render a recoverable configuration error. Co-Authored-By: Claude Opus 5.5 --- .../master-page/master-page.component.ts | 18 +- .../chain-dashboard.component.html | 8 +- .../chain-dashboard.component.ts | 8 +- .../chain-dashboard.service.ts | 6 +- .../chain-mining.component.html | 8 +- .../chain-dashboard/chain-mining.component.ts | 8 +- .../chain-docs/chain-docs.component.ts | 9 +- .../universe/chain-network-consumers.spec.ts | 86 ++++++++ .../src/app/universe/chain-network.spec.ts | 80 +++++++- frontend/src/app/universe/chain-network.ts | 188 ++++++++++++------ .../multichain-explorer.component.ts | 28 ++- .../app/universe/universe-api.service.spec.ts | 51 ++++- .../src/app/universe/universe-api.service.ts | 124 ++++++------ .../app/universe/universe-local.service.ts | 15 +- .../universe/universe-websocket.service.ts | 12 +- .../zcash-viewing-key-workspace.component.ts | 5 +- 16 files changed, 480 insertions(+), 174 deletions(-) create mode 100644 frontend/src/app/universe/chain-network-consumers.spec.ts diff --git a/frontend/src/app/components/master-page/master-page.component.ts b/frontend/src/app/components/master-page/master-page.component.ts index 9b72995f6f..77377a54ef 100644 --- a/frontend/src/app/components/master-page/master-page.component.ts +++ b/frontend/src/app/components/master-page/master-page.component.ts @@ -7,7 +7,7 @@ import { EnterpriseService } from '@app/services/enterprise.service'; import { NavigationService } from '@app/services/navigation.service'; import { StorageService } from '@app/services/storage.service'; import { ChainHealthService, ChainHealthState } from '@app/universe/chain-health.service'; -import { chainNetwork } from '@app/universe/chain-network'; +import { resolveChainNetwork } from '@app/universe/chain-network'; import { healthServiceSummary, nodeHealthLabel, readHealth } from '@app/universe/multichain-explorer/chain-health'; import { UniverseLocalService } from '@app/universe/universe-local.service'; import { mainReady } from '@app/universe/main-ready'; @@ -244,7 +244,8 @@ export class MasterPageComponent implements OnInit, AfterViewInit, OnDestroy { } chainCapability(capabilities: ChainCapabilityEnvelope[], chain: ExplorerChain): ChainCapabilityEnvelope | undefined { - return capabilities.find((capability) => capability.chain === chain && capability.network === this.resolvedNetwork(chain)); + const network = this.resolvedNetwork(chain); + return network === null ? undefined : capabilities.find((capability) => capability.chain === chain && capability.network === network); } chainState(capability: ChainCapabilityEnvelope | undefined): string { @@ -252,12 +253,17 @@ export class MasterPageComponent implements OnInit, AfterViewInit, OnDestroy { } chainNetwork(chain: ExplorerChain): string { - return this.networkLabel(this.resolvedNetwork(chain)); + const network = this.resolvedNetwork(chain); + return network === null ? $localize`:@@master-page.network-setting-invalid:Network setting invalid` : this.networkLabel(network); } - /** Bitcoin follows the selector; every other chain reads its configured network. */ - private resolvedNetwork(chain: ExplorerChain): string { - return chainNetwork(chain, (this.stateService.network || 'mainnet') as ExplorerNetwork, this.stateService.env); + /** + * Bitcoin follows the selector; every other chain reads its configured + * network, or null when that configuration is invalid and nothing is read. + */ + private resolvedNetwork(chain: ExplorerChain): string | null { + const resolved = resolveChainNetwork(chain, (this.stateService.network || 'mainnet') as ExplorerNetwork, this.stateService.env); + return resolved.available ? resolved.network : null; } chainDetail(capability: ChainCapabilityEnvelope | undefined): string { diff --git a/frontend/src/app/universe/chain-dashboard/chain-dashboard.component.html b/frontend/src/app/universe/chain-dashboard/chain-dashboard.component.html index dbc1717538..e225963ab4 100644 --- a/frontend/src/app/universe/chain-dashboard/chain-dashboard.component.html +++ b/frontend/src/app/universe/chain-dashboard/chain-dashboard.component.html @@ -8,6 +8,12 @@

{{ profile.name }} dashboard

+ + +

{{ profile.name }} mining

+ + +

Dogecoin is merged mined: miners work on Litecoin's scrypt proof of work and commit to Dogecoin blocks through AuxPoW, so most Dogecoin blocks are found by Litecoin pools.

@@ -27,7 +33,7 @@

Mining statistics are offered for {{ profile.name }} mainnet only. This explorer is bound to {{ profile.name }} {{ networkLabel }}, where the block collector does not run, so no reading below is available; nothing is invented in its place.

- diff --git a/frontend/src/app/universe/chain-dashboard/chain-mining.component.ts b/frontend/src/app/universe/chain-dashboard/chain-mining.component.ts index 1ce58703d1..740f441ed7 100644 --- a/frontend/src/app/universe/chain-dashboard/chain-mining.component.ts +++ b/frontend/src/app/universe/chain-dashboard/chain-mining.component.ts @@ -49,7 +49,7 @@ import { MiningPoolsView, MiningSummaryView, } from '@app/universe/universe.types'; -import { chainNetwork } from '@app/universe/chain-network'; +import { resolveChainNetwork } from '@app/universe/chain-network'; import { StateService } from '@app/services/state.service'; interface PoolRowReading { @@ -106,6 +106,8 @@ export class ChainMiningComponent implements OnInit { readonly profile: ChainProfile; /** The configured network of this chain, named in the not-offered notice. */ readonly networkLabel: string; + /** Why this chain is not read at all: its configured network is invalid. */ + readonly networkConfigError: string | null; readonly windows = POOL_WINDOWS; readonly window$ = new BehaviorSubject('1w'); vm$: Observable; @@ -122,7 +124,9 @@ export class ChainMiningComponent implements OnInit { ? 'dogecoin' : 'zcash'; this.profile = chainProfile(this.chain); - this.networkLabel = chainNetwork(this.chain, (this.state.network || 'mainnet') as ExplorerNetwork, this.state.env); + const resolved = resolveChainNetwork(this.chain, (this.state.network || 'mainnet') as ExplorerNetwork, this.state.env); + this.networkLabel = resolved.network ?? ''; + this.networkConfigError = resolved.reason; } ngOnInit(): void { diff --git a/frontend/src/app/universe/chain-docs/chain-docs.component.ts b/frontend/src/app/universe/chain-docs/chain-docs.component.ts index f7884dc843..33c153b1ad 100644 --- a/frontend/src/app/universe/chain-docs/chain-docs.component.ts +++ b/frontend/src/app/universe/chain-docs/chain-docs.component.ts @@ -16,7 +16,7 @@ import { docsSectionsFor, } from '@app/universe/chain-docs/chain-docs-content'; import { ExplorerChain, ExplorerNetwork } from '@app/universe/universe.types'; -import { chainNetwork } from '@app/universe/chain-network'; +import { resolveChainNetwork } from '@app/universe/chain-network'; import { StateService } from '@app/services/state.service'; import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pipe'; @@ -37,7 +37,7 @@ export class ChainDocsComponent implements OnInit { readonly chain: Exclude; readonly profile: ChainProfile; /** The network this deployment reads the chain from; the examples below name it. */ - readonly network: ExplorerNetwork; + readonly network: ExplorerNetwork | 'unavailable'; readonly sections: readonly DocsSection[]; /** The section the URL names, or null on the plain /docs route. */ @@ -56,7 +56,10 @@ export class ChainDocsComponent implements OnInit { ? 'dogecoin' : 'zcash'; this.profile = chainProfile(this.chain); - this.network = chainNetwork(this.chain, 'mainnet', state.env); + // The docs name the configured network in their examples; an invalid + // setting is named as such rather than documented as mainnet. + const resolved = resolveChainNetwork(this.chain, 'mainnet', state.env); + this.network = resolved.network ?? 'unavailable'; this.sections = docsSectionsFor(this.chain); } diff --git a/frontend/src/app/universe/chain-network-consumers.spec.ts b/frontend/src/app/universe/chain-network-consumers.spec.ts new file mode 100644 index 0000000000..8c30352b30 --- /dev/null +++ b/frontend/src/app/universe/chain-network-consumers.spec.ts @@ -0,0 +1,86 @@ +import { readFileSync } from 'node:fs'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { firstValueFrom, of, skipWhile } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { StateService } from '@app/services/state.service'; +import { UniverseApiService } from '@app/universe/universe-api.service'; +import { UniverseWebsocketService } from '@app/universe/universe-websocket.service'; +import { UniverseLocalService } from '@app/universe/universe-local.service'; +import { ChainDashboardService } from '@app/universe/chain-dashboard/chain-dashboard.service'; +import { configuredChainNetworks } from '@app/universe/chain-network'; + +// The consumers of UNIVERSE_CHAIN_NETWORKS: an explicit setting that is wrong +// must reach none of them as a mainnet read, and each must say so. + +const ownerKeyStub = { headers: () => ({}), key: null }; + +function state(chainNetworks: unknown, isBrowser = true): StateService { + return { isBrowser, network: '', env: { UNIVERSE_CHAIN_NETWORKS: chainNetworks } } as unknown as StateService; +} + +describe('network configuration consumers', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + beforeEach(() => { configuredChainNetworks({}); warn.mockClear(); }); + afterEach(() => { configuredChainNetworks({}); vi.unstubAllGlobals(); }); + + it('turns an invalid Dogecoin setting into a dashboard configuration error with no request sent', async () => { + const urls: string[] = []; + const env = state({ dogecoin: 'signet' }); + const api = new UniverseApiService({ get: (url: string) => { urls.push(url); return of({}); } } as unknown as HttpClient, env, ownerKeyStub as never); + const dashboards = new ChainDashboardService(api, new UniverseWebsocketService(state({ dogecoin: 'signet' }, false)), { capability$: () => of(null) } as never); + const dashboard = await firstValueFrom(dashboards.dashboard$('dogecoin')); + const pending = await firstValueFrom(dashboards.pending$('dogecoin')); + expect(dashboard).toEqual({ view: null, error: 'network-config-invalid', stale: true }); + expect(pending.payload).toBeNull(); + expect(urls).toEqual([]); + }); + + it('keeps reading a correctly configured chain beside the invalid one', async () => { + const urls: string[] = []; + const env = state({ dogecoin: 'signet', zcash: 'testnet' }); + const api = new UniverseApiService({ get: (url: string) => { urls.push(url); return of({ chain: 'zcash', network: 'testnet' }); } } as unknown as HttpClient, env, ownerKeyStub as never); + const dashboards = new ChainDashboardService(api, new UniverseWebsocketService(state({}, false)), { capability$: () => of(null) } as never); + const dashboard = await firstValueFrom(dashboards.dashboard$('zcash').pipe(skipWhile(value => value.view === null && value.error === null))); + expect(dashboard.error).toBeNull(); + expect(urls).toEqual(['/api/v1/zcash/dashboard?network=testnet']); + }); + + it('opens no live socket for a chain whose setting is invalid', () => { + const sockets: string[] = []; + vi.stubGlobal('WebSocket', class { constructor(url: string) { sockets.push(url); } addEventListener(): void { /* not reached */ } }); + vi.stubGlobal('location', { protocol: 'https:', host: 'explorer.test' }); + const live = new UniverseWebsocketService(state('{"dogecoin":')); + let completed = false; + let failed = false; + live.stream$('dogecoin').subscribe({ complete: () => completed = true, error: () => failed = true }); + expect(sockets).toEqual([]); + expect(completed).toBe(true); + expect(failed).toBe(false); + }); + + it('never files a visit or bookmark under a substitute network', () => { + const store = new Map(); + (globalThis as Record).localStorage = { + getItem: (key: string): string | null => (store.has(key) ? store.get(key) : null), + setItem: (key: string, value: string): void => void store.set(key, value), + removeItem: (key: string): void => void store.delete(key), + }; + const local = new UniverseLocalService(state({ dogecoin: 'testnet3' })); + const entry = { chain: 'dogecoin' as const, kind: 'transaction' as const, value: 'a'.repeat(64), path: '/dogecoin/tx/' + 'a'.repeat(64), label: 'tx' }; + local.recordVisit(entry); + expect(local.recentSnapshot()).toEqual([]); + expect(local.toggleBookmark(entry)).toBe(false); + expect(local.isBookmarked('transaction', 'a'.repeat(64), 'dogecoin')).toBe(false); + // An entry that already names its network keeps it. + local.recordVisit({ ...entry, network: 'testnet' }); + expect(local.recentSnapshot().map(item => item.network)).toEqual(['testnet']); + }); + + it('renders the configuration error on the dashboard and mining pages instead of the generic outage text', () => { + for (const file of ['./chain-dashboard/chain-dashboard.component.html', './chain-dashboard/chain-mining.component.html']) { + const template = readFileSync(new URL(file, import.meta.url), 'utf8'); + expect(template).toMatch(/role="alert" \*ngIf="networkConfigError"/); + expect(template).toContain("vm.viewError !== 'network-config-invalid'"); + } + }); +}); diff --git a/frontend/src/app/universe/chain-network.spec.ts b/frontend/src/app/universe/chain-network.spec.ts index b3557d2a56..908b4f8449 100644 --- a/frontend/src/app/universe/chain-network.spec.ts +++ b/frontend/src/app/universe/chain-network.spec.ts @@ -1,13 +1,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { chainNetwork, configuredChainNetworks } from '@app/universe/chain-network'; +import { + ChainNetworkUnavailableError, + chainNetwork, + configuredChainNetworks, + resolveChainNetwork, +} from '@app/universe/chain-network'; describe('chainNetwork', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); - beforeEach(() => { warn.mockClear(); configuredChainNetworks({}); }); + beforeEach(() => { configuredChainNetworks({}); warn.mockClear(); }); afterEach(() => { configuredChainNetworks({}); }); it('reads mainnet for every non-Bitcoin chain when nothing is configured', () => { - for (const env of [undefined, null, {}, { UNIVERSE_CHAIN_NETWORKS: undefined }, { UNIVERSE_CHAIN_NETWORKS: '' }, { UNIVERSE_CHAIN_NETWORKS: {} }]) { + for (const env of [undefined, null, {}, { UNIVERSE_CHAIN_NETWORKS: undefined }, { UNIVERSE_CHAIN_NETWORKS: '' }, { UNIVERSE_CHAIN_NETWORKS: {} }, { UNIVERSE_CHAIN_NETWORKS: '{}' }]) { expect(chainNetwork('dogecoin', 'signet', env)).toBe('mainnet'); expect(chainNetwork('zcash', 'signet', env)).toBe('mainnet'); expect(chainNetwork('fractal', 'signet', env)).toBe('mainnet'); @@ -24,18 +29,75 @@ describe('chainNetwork', () => { expect(chainNetwork('zcash', 'testnet4', env)).toBe('regtest'); }); - it('parses the JSON-string form and warns once for a value it cannot use', () => { + it('keeps an omitted chain on mainnet beside a configured one', () => { + const env = { UNIVERSE_CHAIN_NETWORKS: { dogecoin: 'testnet' } }; + expect(chainNetwork('zcash', 'signet', env)).toBe('mainnet'); + expect(chainNetwork('fractal', 'signet', env)).toBe('mainnet'); + }); + + it('parses the JSON-string form the Docker environment supplies', () => { expect(chainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: '{"dogecoin":"testnet"}' })).toBe('testnet'); + expect(chainNetwork('fractal', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: '{"fractal":"testnet"}' })).toBe('testnet'); + expect(warn).not.toHaveBeenCalled(); + }); + + it('makes only the chain with an unsupported network unavailable, never mainnet', () => { const env = { UNIVERSE_CHAIN_NETWORKS: '{"dogecoin":"signet","zcash":"testnet"}' }; - expect(chainNetwork('dogecoin', 'mainnet', env)).toBe('mainnet'); + const dogecoin = resolveChainNetwork('dogecoin', 'mainnet', env); + expect(dogecoin.available).toBe(false); + expect(dogecoin.network).toBeNull(); + expect(dogecoin.reason).toMatch(/UNIVERSE_CHAIN_NETWORKS names "signet" for dogecoin/); + expect(() => chainNetwork('dogecoin', 'mainnet', env)).toThrow(ChainNetworkUnavailableError); expect(chainNetwork('zcash', 'mainnet', env)).toBe('testnet'); - expect(chainNetwork('dogecoin', 'mainnet', env)).toBe('mainnet'); + expect(chainNetwork('bitcoin', 'signet', env)).toBe('signet'); + // Memoized: one warning for the setting, not one per read. + resolveChainNetwork('dogecoin', 'mainnet', env); expect(warn).toHaveBeenCalledTimes(1); - expect(String(warn.mock.calls[0][0])).toMatch(/UNIVERSE_CHAIN_NETWORKS .*dogecoin .*signet/); }); - it.each(['{', 'testnet', 7, true, ['testnet'], { dogecoin: 'testnet3' }, { bitcoin: 'signet' }])('ignores %j with a warning', (value) => { - expect(chainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: value })).toBe('mainnet'); + it('refuses a network the owning contract does not list for that chain', () => { + expect(resolveChainNetwork('fractal', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: { fractal: 'regtest' } }).available).toBe(false); + expect(resolveChainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: { dogecoin: 'testnet3' } }).available).toBe(false); + expect(resolveChainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: { dogecoin: 'Testnet' } }).available).toBe(false); + expect(resolveChainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: { dogecoin: 7 } }).available).toBe(false); + }); + + it.each([ + ['{', /is not valid JSON/], + ['testnet', /is not valid JSON/], + [7, /must be a JSON object/], + [true, /must be a JSON object/], + [['testnet'], /must be a JSON object/], + ['[]', /must be a JSON object/], + [{ bitcoin: 'signet' }, /cannot name a Bitcoin network/], + [{ doge: 'testnet' }, /names "doge", which is not a chain/], + [{ dogecoin: 'testnet', litecoin: 'testnet' }, /names "litecoin", which is not a chain/], + ])('makes every configurable chain unavailable for the unreadable map %j', (value, reason) => { + const env = { UNIVERSE_CHAIN_NETWORKS: value }; + for (const chain of ['dogecoin', 'zcash', 'fractal']) { + const resolved = resolveChainNetwork(chain, 'mainnet', env); + expect(resolved.available).toBe(false); + expect(resolved.reason).toMatch(reason); + let failure: unknown; + try { chainNetwork(chain, 'mainnet', env); } catch (error) { failure = error; } + expect(failure).toBeInstanceOf(ChainNetworkUnavailableError); + expect((failure as ChainNetworkUnavailableError).chain).toBe(chain); + } + // Bitcoin never reads this map, so its selector still stands. + expect(chainNetwork('bitcoin', 'signet', env)).toBe('signet'); expect(warn).toHaveBeenCalledTimes(1); }); + + it('names a chain this explorer does not read as unavailable rather than mainnet', () => { + const resolved = resolveChainNetwork('litecoin', 'mainnet', {}); + expect(resolved.available).toBe(false); + expect(resolved.reason).toMatch(/not a chain this explorer reads/); + }); + + it('recovers as soon as the setting is corrected', () => { + expect(resolveChainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: '{"dogecoin":' }).available).toBe(false); + expect(chainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: '{"dogecoin":"testnet"}' })).toBe('testnet'); + expect(resolveChainNetwork('dogecoin', 'mainnet', { UNIVERSE_CHAIN_NETWORKS: { dogecoin: 'signet' } }).available).toBe(false); + expect(chainNetwork('dogecoin', 'mainnet', {})).toBe('mainnet'); + }); }); diff --git a/frontend/src/app/universe/chain-network.ts b/frontend/src/app/universe/chain-network.ts index 09567731b7..cc571aff74 100644 --- a/frontend/src/app/universe/chain-network.ts +++ b/frontend/src/app/universe/chain-network.ts @@ -10,23 +10,80 @@ import { ExplorerChain, ExplorerNetwork } from './universe.types'; */ export const UNIVERSE_CHAIN_NETWORKS_KEY = 'UNIVERSE_CHAIN_NETWORKS'; -/** The networks a non-Bitcoin chain may be configured to. */ +/** + * The chains this map may name and the networks each one serves. + * + * Pinned to the overlay's request contract, backend-apis + * `src/universe-explorer/contracts/explorer-context.ts` (NETWORKS), limited to + * the chains this explorer reads: the two chain pages and Fractal, whose + * CAT20 protocol page reads its sources and activity under chain=fractal. A + * network the overlay would refuse is refused here too, before any request + * is built. Bitcoin is absent on purpose: it follows the selector. + */ +export const CONFIGURABLE_CHAIN_NETWORKS: Readonly> = { + dogecoin: ['mainnet', 'testnet', 'regtest'], + zcash: ['mainnet', 'testnet', 'regtest'], + fractal: ['mainnet', 'testnet'], +}; + +/** Every network any configurable chain accepts. */ export const CHAIN_NETWORK_VALUES: readonly ExplorerNetwork[] = ['mainnet', 'testnet', 'regtest']; -export type ChainNetworkConfig = Readonly>; +/** + * The parsed map. `networks` holds the explicit valid entries; `invalid` + * names each chain whose explicit entry was refused, with the reason; and + * `mapError` is set when the value as a whole could not be read, which makes + * every configurable chain unavailable rather than silently mainnet. + */ +export interface ChainNetworkConfig { + readonly networks: Readonly>; + readonly invalid: Readonly>; + readonly mapError: string | null; +} + +/** + * The network a chain is read from, or why it has none: exactly one of + * `network` and `reason` is set, as `available` says. + */ +export interface ChainNetworkResolution { + readonly available: boolean; + readonly network: ExplorerNetwork | null; + readonly reason: string | null; +} + +/** + * Raised instead of a request when a chain's configured network is invalid. + * Carries the chain and a readable reason so a page can say what is wrong + * and that nothing was read, never a silent mainnet answer. + */ +export class ChainNetworkUnavailableError extends Error { + constructor(readonly chain: string, readonly reason: string) { + super('chain-network-unavailable'); + this.name = 'ChainNetworkUnavailableError'; + } +} + +export function isChainNetworkUnavailable(error: unknown): error is ChainNetworkUnavailableError { + return error instanceof ChainNetworkUnavailableError; +} type Env = { [UNIVERSE_CHAIN_NETWORKS_KEY]?: unknown } | undefined | null; +const EMPTY: ChainNetworkConfig = Object.freeze({ networks: {}, invalid: {}, mapError: null }); + let lastRaw: unknown = undefined; -let lastParsed: ChainNetworkConfig = {}; +let lastParsed: ChainNetworkConfig = EMPTY; /** * The validated chain network map from `env.UNIVERSE_CHAIN_NETWORKS`. * - * Every entry is checked at read time: a chain other than dogecoin/zcash, a - * network outside {@link CHAIN_NETWORK_VALUES}, or an unparseable value is - * dropped with one console warning and that chain reads mainnet. The result - * is memoized on the raw value so a page does not re-warn on every request. + * An absent or empty value is the production default: nothing configured, + * every chain reads mainnet. A value that is present but wrong is kept as an + * error, never dropped: unreadable JSON, a non-object, or a key naming a chain + * this explorer does not configure (a typo for `dogecoin` must not leave + * Dogecoin on mainnet) fails the whole map; an unsupported network fails that + * one chain. The result is memoized on the raw value so a page does not + * re-warn on every request. */ export function configuredChainNetworks(env: Env): ChainNetworkConfig { const raw = env?.[UNIVERSE_CHAIN_NETWORKS_KEY]; @@ -36,81 +93,90 @@ export function configuredChainNetworks(env: Env): ChainNetworkConfig { return lastParsed; } -/** - * IMPLEMENTATION-HANDOFF [M23-NET] | defect F-M23-02 | coverage C-NET-EXPLICIT - * Preparation only, 2026-09-23. Dependency: M23-BASE; coordinate all callers - * of configuredChainNetworks and chainNetwork before changing their contract. - * Verified at 079dc0d79755bc986bfae3288ffe0e479da3e1f6: malformed JSON or an - * explicit unsupported network is dropped here, then chainNetwork defaults - * to mainnet. chain-network.spec.ts deliberately asserts this for Dogecoin - * signet/testnet3. This violates the requested separation of test and mainnet - * contexts; it is not evidence of a transaction having been sent incorrectly. - * Governing requirement: user brief network isolation (lines 41-49), with - * Bitcoin Signet specified by BIP 325; do not infer Signet support for other - * chains from Bitcoin's selector or from a generic list of network names. - * 1. Preserve missing configuration and omitted-chain mainnet defaults, but - * represent an explicitly invalid map/entry as a typed unavailable result - * with a reason. Do not erase the error into an absent entry or substitute a - * different network. Preserve valid entries independently where safe. - * 2. Validate keys against the actual Explorer chain registry and each - * authority's declared supported networks. Pin that registry contract first; - * do not derive support from CHAIN_NETWORK_VALUES alone or invent new chains. - * 3. Update callers to stop affected API requests, clear prior-context data, - * and render a recoverable configuration error. Keep request/cache/query - * identities chain-and-network bound; retry only after valid configuration. - * 4. Extend chain-network.spec.ts with invalid JSON, invalid explicit network, - * unknown chain, valid object/string, omitted defaults, and configuration - * correction. Add consumer tests proving zero wrong-network requests and no - * stale mainnet data after an invalid test-network selection or reconnect. - * Commands (declared, not executed on SERVER in this preparation): - * cd frontend; npm run test:ci -- src/app/universe/chain-network.spec.ts - * npm run lint; npm run build:universe - * Acceptance: explicit invalid input never becomes a mainnet request; absent - * defaults stay mainnet; valid chain-specific testnet reads survive Bitcoin - * selector changes, refresh and reconnect. Record actual supported-network - * consumer evidence separately from unit tests. No mainnet test transactions. - * Rollback: revert the coordinated parser/caller change together; preserve - * production defaults and network-scoped caches. No database migration here. - */ function parse(raw: unknown): ChainNetworkConfig { - if (raw === undefined || raw === null || raw === '') {return {};} + if (raw === undefined || raw === null || raw === '') {return EMPTY;} let value: unknown = raw; if (typeof raw === 'string') { try { value = JSON.parse(raw); } catch { - warn('is not valid JSON; every chain reads mainnet.'); - return {}; + return mapFailure('is not valid JSON.'); } } if (!value || typeof value !== 'object' || Array.isArray(value)) { - warn('must be a JSON object such as {"dogecoin":"testnet"}; every chain reads mainnet.'); - return {}; + return mapFailure('must be a JSON object such as {"dogecoin":"testnet"}.'); } - const parsed: Record = {}; - for (const [chain, network] of Object.entries(value as Record)) { + const entries = Object.entries(value as Record); + for (const [chain] of entries) { if (chain === 'bitcoin') { - warn('cannot name a Bitcoin network; Bitcoin follows the network selector.'); - } else if (typeof network !== 'string' || !CHAIN_NETWORK_VALUES.includes(network as ExplorerNetwork)) { - warn('names an unsupported network for ' + chain + ' (' + String(network) + '); ' + chain + ' reads mainnet.'); + return mapFailure('cannot name a Bitcoin network; Bitcoin follows the network selector.'); + } + if (!hasOwn(CONFIGURABLE_CHAIN_NETWORKS, chain)) { + return mapFailure('names ' + JSON.stringify(chain) + ', which is not a chain this explorer reads.'); + } + } + const networks: Record = {}; + const invalid: Record = {}; + for (const [chain, network] of entries) { + const supported = CONFIGURABLE_CHAIN_NETWORKS[chain as keyof typeof CONFIGURABLE_CHAIN_NETWORKS]; + if (typeof network === 'string' && supported.includes(network as ExplorerNetwork)) { + networks[chain] = network as ExplorerNetwork; } else { - parsed[chain] = network as ExplorerNetwork; + invalid[chain] = UNIVERSE_CHAIN_NETWORKS_KEY + ' names ' + JSON.stringify(network) + ' for ' + chain + + ', which is not one of its networks (' + supported.join(', ') + ').'; + warn(invalid[chain]); } } - return parsed; + return { networks, invalid, mapError: null }; +} + +function mapFailure(detail: string): ChainNetworkConfig { + const reason = UNIVERSE_CHAIN_NETWORKS_KEY + ' ' + detail; + warn(reason); + return { networks: {}, invalid: {}, mapError: reason }; } function warn(detail: string): void { // eslint-disable-next-line no-console - console.warn(UNIVERSE_CHAIN_NETWORKS_KEY + ' ' + detail); + console.warn(detail + ' Reads of the affected chain are stopped until the setting is corrected.'); +} + +/** + * The network a chain is read from, or why it cannot be read: the selected + * Bitcoin network for Bitcoin, the configured network for every other chain, + * mainnet for a chain the map does not name. An explicit entry that is wrong + * never becomes mainnet. The Bitcoin selector never implies another chain's + * network. + */ +export function resolveChainNetwork(chain: ExplorerChain | string, bitcoinNetwork: ExplorerNetwork, env: Env): ChainNetworkResolution { + if (chain === 'bitcoin') {return available(bitcoinNetwork);} + const config = configuredChainNetworks(env); + if (!hasOwn(CONFIGURABLE_CHAIN_NETWORKS, chain)) { + return unavailable(JSON.stringify(chain) + ' is not a chain this explorer reads.'); + } + if (config.mapError) {return unavailable(config.mapError);} + if (hasOwn(config.invalid, chain)) {return unavailable(config.invalid[chain]);} + return available(config.networks[chain] ?? 'mainnet'); +} + +function available(network: ExplorerNetwork): ChainNetworkResolution { + return { available: true, network, reason: null }; +} + +function unavailable(reason: string): ChainNetworkResolution { + return { available: false, network: null, reason }; +} + +function hasOwn(object: object, key: string): boolean { + return Object.prototype.hasOwnProperty.call(object, key); } /** - * The network a chain is read from: the selected Bitcoin network for Bitcoin, - * the configured network for every other chain. The Bitcoin selector never - * implies another chain's network. + * {@link resolveChainNetwork} for a caller about to build a request: the + * network, or a {@link ChainNetworkUnavailableError} so no request is sent. */ export function chainNetwork(chain: ExplorerChain | string, bitcoinNetwork: ExplorerNetwork, env: Env): ExplorerNetwork { - if (chain === 'bitcoin') {return bitcoinNetwork;} - return configuredChainNetworks(env)[chain] ?? 'mainnet'; + const resolved = resolveChainNetwork(chain, bitcoinNetwork, env); + if (!resolved.available || !resolved.network) {throw new ChainNetworkUnavailableError(chain, resolved.reason ?? 'unavailable');} + return resolved.network; } diff --git a/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts b/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts index c9bb68e44b..f3151943c5 100644 --- a/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts +++ b/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts @@ -17,6 +17,7 @@ import { } from '@angular/router'; import { SeoService } from '@app/services/seo.service'; import { UniverseApiService } from '@app/universe/universe-api.service'; +import { isChainNetworkUnavailable } from '@app/universe/chain-network'; import { UniverseWebsocketService } from '@app/universe/universe-websocket.service'; import { UniverseEntryKind, @@ -96,6 +97,7 @@ import { Subject, catchError, combineLatest, + defer, map, of, switchMap, @@ -453,9 +455,13 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { if (!kind || !this.reference) { return; } + const network = this.api.chainNetworkLabel(this.chain); + if (!network) { + return; + } this.saved = this.local.toggleBookmark({ chain: this.chain, - network: this.api.chainNetwork(this.chain), + network, kind, value: this.reference, path: this.router.url.split('?')[0], @@ -727,12 +733,12 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { reference, 'holders' ) - : of({ + : defer(() => of({ chain: this.chain, network: this.api.chainNetwork(this.chain), state: 'unavailable', reason: 'section-not-supported', - }); + })); case 'protocol-events': return this.chain === 'dogecoin' ? this.api.getChainProtocolSection$( @@ -741,12 +747,12 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { reference, 'events' ) - : of({ + : defer(() => of({ chain: this.chain, network: this.api.chainNetwork(this.chain), state: 'unavailable', reason: 'section-not-supported', - }); + })); } } @@ -759,12 +765,13 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { private recordVisit(context: RequestContext): void { const kind = PAGE_KINDS[context.page]; - if (!kind || !this.reference) { + const network = this.api.chainNetworkLabel(this.chain); + if (!kind || !this.reference || !network) { return; } this.local.recordVisit({ chain: this.chain, - network: this.api.chainNetwork(this.chain), + network, kind, value: this.reference, path: this.router.url.split('?')[0], @@ -774,10 +781,12 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { private isSaved(context: RequestContext): boolean { const kind = PAGE_KINDS[context.page]; + const network = this.api.chainNetworkLabel(this.chain); return ( !!kind && !!this.reference && - this.local.isBookmarked(kind, this.reference, this.chain, this.api.chainNetwork(this.chain)) + !!network && + this.local.isBookmarked(kind, this.reference, this.chain, network) ); } @@ -821,6 +830,9 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { } private errorMessage(error: unknown): string { + if (isChainNetworkUnavailable(error)) { + return $localize`:@@universe.chain.error-network-config:${this.chainName}:CHAIN: is not being read because its network setting is invalid: ${error.reason}:REASON: Nothing from another network is shown. Correct the setting, then reload.`; + } const status = typeof error === 'object' && error !== null && 'status' in error ? String((error as { status: unknown }).status) diff --git a/frontend/src/app/universe/universe-api.service.spec.ts b/frontend/src/app/universe/universe-api.service.spec.ts index a1bc7672fe..ed35b24d41 100644 --- a/frontend/src/app/universe/universe-api.service.spec.ts +++ b/frontend/src/app/universe/universe-api.service.spec.ts @@ -3,7 +3,7 @@ import { BehaviorSubject, Observable, Subject, of, throwError } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { StateService } from '@app/services/state.service'; import { UniverseApiService } from '@app/universe/universe-api.service'; -import { configuredChainNetworks } from '@app/universe/chain-network'; +import { ChainNetworkUnavailableError, configuredChainNetworks } from '@app/universe/chain-network'; // Owner-scoped calls read a bearer header from this; the specs here make none. const ownerKeyStub = { headers: () => ({}), key: null }; @@ -267,14 +267,53 @@ describe('UniverseApiService chain network context', () => { }); it.each([ - { dogecoin: 'signet' }, { dogecoin: 'Testnet' }, { dogecoin: 7 }, 'not json', ['testnet'], { bitcoin: 'testnet' }, - ])('ignores an invalid configuration %j with a warning and reads mainnet', (value) => { + { dogecoin: 'signet' }, { dogecoin: 'Testnet' }, { dogecoin: 7 }, 'not json', ['testnet'], { bitcoin: 'testnet' }, { doge: 'testnet' }, + ])('sends no Dogecoin request for the invalid configuration %j and fails with the typed reason', (value) => { const { service, urls } = build(true, undefined, value); - service.getChainDashboard$('dogecoin').subscribe(); - expect(urls).toEqual(['/api/v1/dogecoin/dashboard?network=mainnet']); + const failures: unknown[] = []; + const reads = [ + service.getChainDashboard$('dogecoin'), service.getChainMempool$('dogecoin', 10), service.getChainCandidateBuckets$('dogecoin'), + service.getChainRecentBlocks$('dogecoin'), service.getChainFees$('dogecoin'), service.getChainMining$('dogecoin'), + service.getChainMiningPools$('dogecoin'), service.getChainChartSeries$('dogecoin', 'block-fees'), + service.getChainTransaction$('dogecoin', 'a'.repeat(64)), service.getChainBlock$('dogecoin', '1'), + service.getChainAddress$('dogecoin', 'D'), service.getChainAddressHoldings$('dogecoin', 'D'), + service.getChainOutpoint$('dogecoin', 'a'.repeat(64), '0'), service.getChainProtocols$('dogecoin'), + service.getChainProtocolList$('dogecoin', 'drc20'), service.getChainProtocolDetail$('dogecoin', 'drc20', 'tick'), + service.getChainProtocolSection$('dogecoin', 'drc20', 'tick', 'holders'), service.getChainStatus$('dogecoin'), + service.getSources$('dogecoin'), service.search$('abc', 'dogecoin'), + ]; + for (const read of reads) {read.subscribe({ error: (error) => failures.push(error) });} + expect(urls).toEqual([]); + expect(failures).toHaveLength(reads.length); + for (const failure of failures) { + expect(failure).toBeInstanceOf(ChainNetworkUnavailableError); + expect((failure as ChainNetworkUnavailableError).reason).toContain('UNIVERSE_CHAIN_NETWORKS'); + } + expect(service.chainNetworkLabel('dogecoin')).toBeNull(); expect(service.chainNetwork('bitcoin')).toBe('mainnet'); expect(warn).toHaveBeenCalledTimes(1); - expect(String(warn.mock.calls[0][0])).toContain('UNIVERSE_CHAIN_NETWORKS'); + }); + + it('leaves an invalid chain out of the picker read and keeps the other chains on their own network', () => { + const { service, urls } = build(true, undefined, { dogecoin: 'signet' }); + let rows: unknown[] = []; + service.getChains$().subscribe(value => rows = value); + expect(urls).toEqual(['/api/v1/chains/bitcoin?network=mainnet', '/api/v1/chains/zcash?network=mainnet']); + expect(rows).toEqual([{ chain: 'bitcoin', network: 'mainnet' }, { chain: 'zcash', network: 'mainnet' }]); + }); + + it('resolves the network again on retry, so a corrected setting is read without a stale substitute', () => { + const state = { isBrowser: true, network: '', env: { UNIVERSE_CHAIN_NETWORKS: '{"dogecoin":' } } as unknown as StateService; + const urls: string[] = []; + const service = new UniverseApiService({ get: (url: string) => { urls.push(url); return of({}); } } as unknown as HttpClient, state, ownerKeyStub as never); + const read = service.getChainFees$('dogecoin'); + let failure: unknown; + read.subscribe({ error: (error) => failure = error }); + expect(failure).toBeInstanceOf(ChainNetworkUnavailableError); + expect(urls).toEqual([]); + (state.env as Record).UNIVERSE_CHAIN_NETWORKS = '{"dogecoin":"testnet"}'; + read.subscribe(); + expect(urls).toEqual(['/api/v1/dogecoin/fees?network=testnet']); }); it('never lets a mainnet capability record stand in for a configured testnet scope', () => { diff --git a/frontend/src/app/universe/universe-api.service.ts b/frontend/src/app/universe/universe-api.service.ts index ccaf276026..21dd1f1549 100644 --- a/frontend/src/app/universe/universe-api.service.ts +++ b/frontend/src/app/universe/universe-api.service.ts @@ -4,7 +4,7 @@ import { Observable, catchError, forkJoin, map, of, shareReplay, throwError, def import { TransactionAssetSummary, decodeTransactionAssetSummary } from './transaction-assets/transaction-assets.types'; import { StateService } from '@app/services/state.service'; import { ProtocolPageKind, readProtocolFailure, readProtocolPage } from './universe-protocol-contract'; -import { chainNetwork } from './chain-network'; +import { chainNetwork, resolveChainNetwork } from './chain-network'; import { BackendInfo, ExplorerTransactionAssetFlow, @@ -177,12 +177,24 @@ export class UniverseApiService { * and the configured UNIVERSE_CHAIN_NETWORKS entry (mainnet when unlisted) * for every other chain. The Bitcoin selector never implies another chain's * network, so a Signet reader still reads Dogecoin from its configured network. + * Throws a ChainNetworkUnavailableError when the chain's entry is invalid; + * use {@link chainNetworkLabel} where a page only names the network. */ chainNetwork(chain: ExplorerChain | string): ExplorerNetwork { return chainNetwork(chain, this.network, this.stateService.env); } - /** {@link chainNetwork} as a stream: re-emits only when the Bitcoin selection matters. */ + /** The network a page names for a chain, or null when its configuration is invalid. */ + chainNetworkLabel(chain: ExplorerChain | string): ExplorerNetwork | null { + const resolved = resolveChainNetwork(chain, this.network, this.stateService.env); + return resolved.available ? resolved.network : null; + } + + /** + * {@link chainNetwork} as a stream: re-emits only when the Bitcoin selection + * matters, and errors with a ChainNetworkUnavailableError, before any + * request, when the chain's configured network is invalid. + */ chainNetwork$(chain: ExplorerChain | string): Observable { return this.selectedNetwork$().pipe( map((network) => chainNetwork(chain, network, this.stateService.env)), @@ -190,6 +202,16 @@ export class UniverseApiService { ); } + /** + * One read of a non-Bitcoin chain route under that chain's configured + * network. Deferred so the network is resolved at subscription: an invalid + * configuration fails as a ChainNetworkUnavailableError and sends nothing, + * and a retry resolves again rather than reusing an earlier network. + */ + private chainRead(chain: Exclude, path: (network: ExplorerNetwork) => string): Observable { + return defer(() => this.httpClient.get(this.apiBaseUrl + '/api/v1/' + chain + path(this.chainNetwork(chain)))); + } + private requestForNetwork(url: string, network: ExplorerNetwork, body?: unknown, chain = 'bitcoin'): Observable { const address = url + (url.includes('?') ? '&' : '?') + 'chain=' + encodeURIComponent(chain) + '&network=' + network; const request = body === undefined ? this.httpClient.get(address) : this.httpClient.post(address, body); @@ -383,16 +405,20 @@ export class UniverseApiService { * unavailable record under that network, never a mainnet one. */ getChains$(): Observable { - return this.selectedNetwork$().pipe(switchMap(network => forkJoin(EXPLORER_CHAINS.map(chain => { - const expected = chainNetwork(chain, network, this.stateService.env); - return this.httpClient.get( + return this.selectedNetwork$().pipe(switchMap(network => forkJoin(EXPLORER_CHAINS.flatMap(chain => { + const resolved = resolveChainNetwork(chain, network, this.stateService.env); + // An invalid configuration is named by the picker itself; asking the + // overlay under a substitute network would show another network's health. + if (!resolved.available) {return [];} + const expected = resolved.network; + return [this.httpClient.get( this.apiBaseUrl + '/api/v1/chains/' + chain + '?network=' + expected, ).pipe(map(row => { if (!row || row.chain !== chain) {throw new Error('invalid-chain-capabilities');} if (row.network !== expected) {throw new Error('authority-network-mismatch');} this.assertResponseContext(row, expected, chain); return row; - })); + }))]; })))); } @@ -430,78 +456,56 @@ export class UniverseApiService { } getChainMempool$(chain: Exclude, limit = 100): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/mempool?network=' + this.chainNetwork(chain) + '&limit=' - + Math.min(Math.max(1, Math.floor(limit)), CHAIN_MEMPOOL_LIMIT[chain]) - ); + return this.chainRead(chain, (network) => '/mempool?network=' + network + '&limit=' + + Math.min(Math.max(1, Math.floor(limit)), CHAIN_MEMPOOL_LIMIT[chain])); } getChainCandidateBuckets$(chain: Exclude): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/candidate-buckets?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/candidate-buckets?network=' + network); } /** The one-call dashboard aggregate: blocks, buckets, fees, mempool, mining. */ getChainDashboard$(chain: Exclude): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/dashboard?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/dashboard?network=' + network); } getChainRecentBlocks$(chain: Exclude, limit = 15): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/blocks/recent?network=' + this.chainNetwork(chain) + '&limit=' + limit - ); + return this.chainRead(chain, (network) => '/blocks/recent?network=' + network + '&limit=' + limit); } getChainFees$(chain: Exclude): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/fees?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/fees?network=' + network); } getChainMining$(chain: Exclude): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/mining?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/mining?network=' + network); } getChainMiningPools$(chain: Exclude, window = '1w'): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/mining/pools?network=' + this.chainNetwork(chain) + '&window=' + encodeURIComponent(window) - ); + return this.chainRead(chain, (network) => '/mining/pools?network=' + network + '&window=' + encodeURIComponent(window)); } getChainChartSeries$(chain: Exclude, seriesId: string, range = '1w'): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/charts/' + encodeURIComponent(seriesId) - + '?network=' + this.chainNetwork(chain) + '&range=' + encodeURIComponent(range) - ); + return this.chainRead(chain, (network) => '/charts/' + encodeURIComponent(seriesId) + + '?network=' + network + '&range=' + encodeURIComponent(range)); } getChainTransaction$(chain: Exclude, txid: string): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/tx/' + encodeURIComponent(txid) + '?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/tx/' + encodeURIComponent(txid) + '?network=' + network); } getChainBlock$(chain: Exclude, reference: string, limit = 100, offset = 0): Observable { const paging = chain === 'dogecoin' ? '&page=' + (Math.floor(offset / limit) + 1) + '&limit=' + limit : '&limit=' + limit + '&offset=' + offset; - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/block/' + encodeURIComponent(reference) + '?network=' + this.chainNetwork(chain) + paging - ); + return this.chainRead(chain, (network) => '/block/' + encodeURIComponent(reference) + '?network=' + network + paging); } getChainAddress$(chain: Exclude, address: string, limit = 100, offset = 0): Observable { const paging = chain === 'dogecoin' ? '&page=' + (Math.floor(offset / limit) + 1) + '&limit=' + limit : '&limit=' + limit + '&offset=' + offset; - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/address/' + encodeURIComponent(address) + '?network=' + this.chainNetwork(chain) + paging - ); + return this.chainRead(chain, (network) => '/address/' + encodeURIComponent(address) + '?network=' + network + paging); } /** @@ -511,9 +515,8 @@ export class UniverseApiService { * degrades the asset sections without taking the address page down. */ getChainAddressHoldings$(chain: Exclude, address: string, limit = 50, offset = 0): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/address/' + encodeURIComponent(address) + '/holdings?network=' + this.chainNetwork(chain) + '&limit=' + limit + '&offset=' + offset - ); + return this.chainRead(chain, (network) => '/address/' + encodeURIComponent(address) + '/holdings?network=' + network + + '&limit=' + limit + '&offset=' + offset); } /** The Bitcoin address asset-holdings view from the universe overlay. */ @@ -524,38 +527,30 @@ export class UniverseApiService { } getChainOutpoint$(chain: Exclude, txid: string, vout: string): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/outpoint/' + encodeURIComponent(txid) + '/' + encodeURIComponent(vout) + '?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/outpoint/' + encodeURIComponent(txid) + '/' + encodeURIComponent(vout) + + '?network=' + network); } getChainProtocols$(chain: Exclude): Observable { - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/protocols?network=' + this.chainNetwork(chain) - ); + return this.chainRead(chain, (network) => '/protocols?network=' + network); } getChainProtocolList$(chain: Exclude, protocol: string, limit = 100, offset = 0, ruleset?: string): Observable { const path = this.protocolPath(chain, protocol); - let query = '?network=' + this.chainNetwork(chain) + '&limit=' + limit; + let paging = '&limit=' + limit; if (chain === 'dogecoin' && protocol !== 'doge-tap') { - query += '&cursor=' + offset; + paging += '&cursor=' + offset; } else if (chain === 'dogecoin') { - query += '&offset=' + offset; + paging += '&offset=' + offset; } - if (ruleset) {query += '&ruleset=' + encodeURIComponent(ruleset);} - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/protocols/' + path + query - ); + if (ruleset) {paging += '&ruleset=' + encodeURIComponent(ruleset);} + return this.chainRead(chain, (network) => '/protocols/' + path + '?network=' + network + paging); } getChainProtocolDetail$(chain: Exclude, protocol: string, reference: string, ruleset?: string): Observable { const path = this.protocolPath(chain, protocol); - let query = '?network=' + this.chainNetwork(chain); - if (ruleset) {query += '&ruleset=' + encodeURIComponent(ruleset);} - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/protocols/' + path + '/' + encodeURIComponent(reference) + query - ); + const extra = ruleset ? '&ruleset=' + encodeURIComponent(ruleset) : ''; + return this.chainRead(chain, (network) => '/protocols/' + path + '/' + encodeURIComponent(reference) + '?network=' + network + extra); } getChainProtocolSection$(chain: 'dogecoin', protocol: string, reference: string, section: 'holders' | 'events', limit = 100, offset = 0): Observable { @@ -563,9 +558,8 @@ export class UniverseApiService { const paging = protocol === 'drc20' ? '&cursor=' + offset : '&offset=' + offset; - return this.httpClient.get( - this.apiBaseUrl + '/api/v1/' + chain + '/protocols/' + path + '/' + encodeURIComponent(reference) + '/' + section + '?network=' + this.chainNetwork(chain) + '&limit=' + limit + paging - ); + return this.chainRead(chain, (network) => '/protocols/' + path + '/' + encodeURIComponent(reference) + '/' + section + + '?network=' + network + '&limit=' + limit + paging); } diff --git a/frontend/src/app/universe/universe-local.service.ts b/frontend/src/app/universe/universe-local.service.ts index dffe67d6a4..48477861ed 100644 --- a/frontend/src/app/universe/universe-local.service.ts +++ b/frontend/src/app/universe/universe-local.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { BehaviorSubject, Observable } from 'rxjs'; import { StateService } from '@app/services/state.service'; import { ExplorerChain, ExplorerNetwork } from '@app/universe/universe.types'; -import { chainNetwork } from '@app/universe/chain-network'; +import { resolveChainNetwork } from '@app/universe/chain-network'; /** * Local personalization for the explorer. @@ -107,8 +107,10 @@ export class UniverseLocalService { private newEntry(entry: UniverseEntryInput): UniverseEntry | null { // Only new writes use current navigation state. Historical entries use their own path/provenance. - const network = entry.network ?? chainNetwork(entry.chain ?? 'bitcoin', this.currentNetwork(), this.stateService.env); - return this.sanitizeEntry({ ...entry, network, at: Date.now() }); + if (entry.network) {return this.sanitizeEntry({ ...entry, at: Date.now() });} + // An entry is never filed under a network nothing was read from. + const resolved = resolveChainNetwork(entry.chain ?? 'bitcoin', this.currentNetwork(), this.stateService.env); + return resolved.available ? this.sanitizeEntry({ ...entry, network: resolved.network, at: Date.now() }) : null; } private read(key: string): unknown { @@ -233,8 +235,13 @@ export class UniverseLocalService { kind: UniverseEntryKind, value: string, chain: ExplorerChain = 'bitcoin', - network: ExplorerNetwork = chainNetwork(chain, this.currentNetwork(), this.stateService.env), + network?: ExplorerNetwork, ): boolean { + if (!network) { + const resolved = resolveChainNetwork(chain, this.currentNetwork(), this.stateService.env); + if (!resolved.available) {return false;} + network = resolved.network; + } const id = entryKey({ chain, network, kind, value }); return this.bookmarkSubject.value.some((item) => entryKey(item) === id); } diff --git a/frontend/src/app/universe/universe-websocket.service.ts b/frontend/src/app/universe/universe-websocket.service.ts index caefd27892..58966a1bd4 100644 --- a/frontend/src/app/universe/universe-websocket.service.ts +++ b/frontend/src/app/universe/universe-websocket.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { StateService } from '@app/services/state.service'; import { ExplorerChain, ExplorerNetwork } from '@app/universe/universe.types'; -import { chainNetwork } from '@app/universe/chain-network'; +import { resolveChainNetwork } from '@app/universe/chain-network'; import { EMPTY, Observable } from 'rxjs'; export interface UniverseLiveEnvelope { @@ -72,7 +72,15 @@ export class UniverseWebsocketService { } // Bitcoin live frames stay on mainnet as before; every other chain // subscribes to and accepts only its configured network. - const network = chainNetwork(chain, 'mainnet', this.stateService.env); + const resolved = resolveChainNetwork(chain, 'mainnet', this.stateService.env); + // No socket is opened for a chain whose configured network is invalid: + // subscribing under a substitute network would stream another network. + // The stream stays silent rather than failing, so a page polling beside it + // keeps running and shows the configuration error its own reads raise. + if (!resolved.available) { + return EMPTY; + } + const network = resolved.network; return new Observable((observer) => { const cursors = new Map(); let socket: WebSocket | null = null; diff --git a/frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts b/frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts index f6737a19e2..468c51f763 100644 --- a/frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts +++ b/frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts @@ -8,20 +8,21 @@ import { StateService } from '@app/services/state.service'; import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pipe'; import { ZcashScannerService, ZcashScanResult } from './zcash-scanner.service'; import { ZCASH_SCANNER_SAMPLES } from './zcash-scanner-samples'; -import { chainNetwork } from '../chain-network'; +import { resolveChainNetwork } from '../chain-network'; @Component({selector:'app-zcash-viewing-key-workspace',templateUrl:'./zcash-viewing-key-workspace.component.html',styleUrls:['../product-page.scss'],standalone:true,imports:[RelativeUrlPipe,CommonModule,FormsModule,RouterModule],changeDetection:ChangeDetectionStrategy.OnPush}) export class ZcashViewingKeyWorkspaceComponent implements OnDestroy { viewingKey='';keyType='unified-full';network='mainnet';mode='owned-blocks';startHeight=2500000;blockCount=1;artifactInput='[]';scanning=false;error:string|null=null; private resultSubject=new BehaviorSubject(null);readonly result$=this.resultSubject.asObservable(); private active?:{promise:Promise;cancel:()=>void};private abort?:AbortController;private generation=0;private networkSubscription:Subscription; resume:{hash:string;height:number}|null=null; - constructor(seo:SeoService,private scanner:ZcashScannerService,private cdr:ChangeDetectorRef,state:StateService){seo.setTitle('Zcash Client-Only Viewing-Key Workspace');this.network=chainNetwork('zcash','mainnet',state.env);this.networkSubscription=state.networkChanged$.subscribe(()=>this.clear());} + constructor(seo:SeoService,private scanner:ZcashScannerService,private cdr:ChangeDetectorRef,state:StateService){seo.setTitle('Zcash Client-Only Viewing-Key Workspace');const resolved=resolveChainNetwork('zcash','mainnet',state.env);this.network=resolved.network??'';this.error=resolved.reason?resolved.reason+' Choose a Zcash network before scanning.':null;this.networkSubscription=state.networkChanged$.subscribe(()=>this.clear());} clear(resetResume=true){this.generation++;this.active?.cancel();this.abort?.abort();this.scanning=false;this.error=null;this.resultSubject.next(null);if(resetResume)this.resume=null;this.cdr.markForCheck();} clearKey(){this.clear();this.viewingKey='';} loadSample(pool:'sapling'|'orchard'){this.clear();const sample=ZCASH_SCANNER_SAMPLES[pool];this.network=sample.network;this.mode=sample.mode;this.keyType=sample.key_type;this.viewingKey=sample.viewing_key;this.artifactInput=JSON.stringify(sample.outputs,null,2);} async scan(resuming=false){ const prior=resuming?this.resume:null;this.clear(false);this.resume=null; if(!this.viewingKey.trim()){this.error='A supported viewing key is required.';return;} + if(this.network!=='mainnet'&&this.network!=='testnet'){this.error='Choose a Zcash network before scanning.';return;} if(resuming&&!prior){this.error='No verified interval checkpoint is available.';return;} const generation=this.generation;this.scanning=true; const key={viewing_key:this.viewingKey.trim(),key_type:this.keyType,network:this.network}; From 6f571edc7e8f0e383b6742a19554680b2718f7a5 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 20:57:02 +0000 Subject: [PATCH 06/11] fix(capabilities): judge mining readiness against a fresh same-network Core reading M23-HEALTH, F-M23-03. Mining was reported ready whenever the blocks and pools tables had rows; production published ready with the index at 968172 and Core at 968299. Readiness now compares the highest indexed block with Core's height, requiring the reading to be under 120 s old and from the served network, within MEMPOOL.MINING_MAX_BEHIND_TIP (default 3). A missing, stale or other-network reading is the new unknown state; Core in initial block download is syncing; an index ahead of Core after a reorganization withholds readiness. The report now carries indexedTip, bitcoinCoreTip, lagBlocks and maxLagBlocks. backend-info records Core's chain name so the comparison can be matched to a network. Co-Authored-By: Claude Opus 5.5 --- backend/mempool-config.sample.json | 1 + .../__fixtures__/mempool-config.template.json | 1 + .../capabilities-mining-report.test.ts | 144 ++++++++++++++++++ .../src/__tests__/capabilities-mining.test.ts | 98 ++++++++++++ backend/src/__tests__/config.test.ts | 1 + backend/src/api/backend-info.ts | 1 + backend/src/api/capabilities.mining.ts | 112 ++++++++++++++ backend/src/api/capabilities.routes.ts | 49 +----- backend/src/api/capabilities.ts | 41 ++++- backend/src/config.ts | 2 + backend/src/mempool.interfaces.ts | 5 + docker/backend/mempool-config.json | 1 + docker/backend/start.sh | 2 + docs/operations/CONFIGURATION.md | 19 ++- 14 files changed, 420 insertions(+), 57 deletions(-) create mode 100644 backend/src/__tests__/capabilities-mining-report.test.ts create mode 100644 backend/src/__tests__/capabilities-mining.test.ts create mode 100644 backend/src/api/capabilities.mining.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c1250f4466..64864ce4ba 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -30,6 +30,7 @@ "POOLS_JSON_TREE_URL": "", "POOLS_JSON_FILE": "tasks/pools/pools-v2.json", "POOLS_UPDATE_DELAY": 604800, + "MINING_MAX_BEHIND_TIP": 3, "AUDIT": false, "RUST_GBT": true, "LIMIT_GBT": false, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 7f56f841fd..1976e842f4 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -31,6 +31,7 @@ "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "POOLS_JSON_FILE": "__MEMPOOL_POOLS_JSON_FILE__", "POOLS_UPDATE_DELAY": 604800, + "MINING_MAX_BEHIND_TIP": 3, "AUDIT": true, "RUST_GBT": false, "LIMIT_GBT": false, diff --git a/backend/src/__tests__/capabilities-mining-report.test.ts b/backend/src/__tests__/capabilities-mining-report.test.ts new file mode 100644 index 0000000000..c51ea043b1 --- /dev/null +++ b/backend/src/__tests__/capabilities-mining-report.test.ts @@ -0,0 +1,144 @@ +/** + * The mining section of /api/v1/capabilities, end to end through $report with + * the database and the Core reading replaced. The verdict itself is covered in + * capabilities-mining.test.ts; these prove the report feeds it the right facts + * and that a cached answer cannot outlive the state it described. + */ + +interface BlockRow { total: number; lowest: number | null; highest: number | null; newest: Date | null } +const db: { up: boolean; blocks: BlockRow; pools: number } = { + up: true, + blocks: { total: 11_000, lowest: 957_300, highest: 968_299, newest: new Date('2026-09-23T17:55:00.000Z') }, + pools: 180, +}; +const sync: { value: Record | null } = { value: null }; + +jest.mock('../database', () => ({ + __esModule: true, + default: { + query: async (sql: string) => { + if (!db.up) {throw new Error('connect ECONNREFUSED');} + if (sql.includes('SELECT 1')) {return [[{ 1: 1 }]];} + if (sql.includes('FROM blocks')) {return [[db.blocks]];} + if (sql.includes('FROM pools')) {return [[{ total: db.pools }]];} + if (sql.includes('FROM statistics')) {return [[{ total: 1, oldest: new Date(), newest: new Date() }]];} + throw new Error('unexpected query ' + sql); + }, + }, +})); +jest.mock('../api/backend-info', () => ({ + __esModule: true, + default: { getBackendInfo: () => ({ gitCommit: 'test', chainSync: sync.value }) }, +})); +jest.mock('../api/capabilities.optional', () => ({ $optionalCapabilityReports: async () => ({}) })); +jest.mock('../api/bitcoin/address-index', () => ({ + addressBackendKind: () => 'electrum', + $probeAddressIndex: async () => ({ + configured: true, reachable: true, summaryAnswered: true, utxoAnswered: true, state: 'ready', + degradedReason: null, backendKind: 'electrum', indexedTip: null, chainTip: null, lagBlocks: null, + maxBehindTip: 2, sourceRelease: null, + }), +})); + +import config from '../config'; +import capabilities from '../api/capabilities'; + +function freshCore(blocks: number, chain = 'main'): Record { + return { blocks, headers: blocks, initialBlockDownload: false, verificationProgress: 1, checkedAt: new Date().toISOString(), chain }; +} + +async function mining(): Promise> { + // Each case asks a fresh question; the ten second cache is exercised on its own below. + (capabilities as unknown as { cached: unknown }).cached = null; + return (await capabilities.$report()).features.mining as unknown as Record; +} + +describe('mining capability report', () => { + const saved = { indexing: config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, enabled: config.MEMPOOL.ENABLED, database: config.DATABASE.ENABLED, network: config.MEMPOOL.NETWORK }; + + beforeAll(() => { + config.MEMPOOL.INDEXING_BLOCKS_AMOUNT = 11_000; + config.MEMPOOL.ENABLED = true; + config.DATABASE.ENABLED = true; + config.MEMPOOL.NETWORK = 'mainnet'; + capabilities.markRoutesRegistered('mining'); + }); + afterAll(() => { + config.MEMPOOL.INDEXING_BLOCKS_AMOUNT = saved.indexing; + config.MEMPOOL.ENABLED = saved.enabled; + config.DATABASE.ENABLED = saved.database; + config.MEMPOOL.NETWORK = saved.network; + }); + beforeEach(() => { + db.up = true; + db.blocks = { total: 11_000, lowest: 957_300, highest: 968_299, newest: new Date('2026-09-23T17:55:00.000Z') }; + db.pools = 180; + sync.value = freshCore(968_299); + }); + + it('publishes the indexed tip, Core tip and lag with a ready verdict when current', async () => { + expect(await mining()).toMatchObject({ + state: 'ready', indexedTip: 968_299, bitcoinCoreTip: 968_299, lagBlocks: 0, + maxLagBlocks: config.MEMPOOL.MINING_MAX_BEHIND_TIP, degradedReason: null, rowCount: 11_000, + coverage: { from: '957300', to: '968299' }, + }); + }); + + it('reports the lagged production state as degraded with its numbers', async () => { + db.blocks = { ...db.blocks, highest: 968_172 }; + expect(await mining()).toMatchObject({ state: 'degraded', indexedTip: 968_172, bitcoinCoreTip: 968_299, lagBlocks: 127 }); + }); + + it('reports unknown, not ready, when Core has never been read', async () => { + sync.value = null; + expect(await mining()).toMatchObject({ state: 'unknown', bitcoinCoreTip: null, lagBlocks: null }); + }); + + it('reports unknown when the Core reading has expired', async () => { + sync.value = { ...freshCore(968_299), checkedAt: new Date(Date.now() - 10 * 60_000).toISOString() }; + expect((await mining()).state).toBe('unknown'); + }); + + it('reports unknown when Core is on another network than the one served', async () => { + sync.value = freshCore(968_299, 'signet'); + expect((await mining()).state).toBe('unknown'); + }); + + it('reports the database loss as unavailable', async () => { + db.up = false; + expect(await mining()).toMatchObject({ state: 'unavailable', degradedReason: 'The mining index database is unavailable.' }); + }); + + it('reports empty tables and missing pool metadata as degraded', async () => { + db.blocks = { total: 0, lowest: null, highest: null, newest: null }; + expect((await mining()).state).toBe('degraded'); + db.blocks = { total: 11_000, lowest: 957_300, highest: 968_299, newest: new Date() }; + db.pools = 0; + expect(await mining()).toMatchObject({ state: 'degraded', degradedReason: 'Mining pool metadata has not been imported yet.' }); + }); + + it('recovers to ready once the collector catches up', async () => { + db.blocks = { ...db.blocks, highest: 968_000 }; + expect((await mining()).state).toBe('degraded'); + db.blocks = { ...db.blocks, highest: 968_299 }; + expect((await mining()).state).toBe('ready'); + }); + + it('never serves a cached ready answer past its ten second life', async () => { + const now = jest.spyOn(Date, 'now'); + try { + const start = Date.parse('2026-09-23T18:00:00.000Z'); + now.mockReturnValue(start); + sync.value = { ...freshCore(968_299), checkedAt: new Date(start).toISOString() }; + (capabilities as unknown as { cached: unknown }).cached = null; + expect((await capabilities.$report()).features.mining.state).toBe('ready'); + db.up = false; + now.mockReturnValue(start + 5_000); + expect((await capabilities.$report()).features.mining.state).toBe('ready'); + now.mockReturnValue(start + 10_001); + expect((await capabilities.$report()).features.mining.state).toBe('unavailable'); + } finally { + now.mockRestore(); + } + }); +}); diff --git a/backend/src/__tests__/capabilities-mining.test.ts b/backend/src/__tests__/capabilities-mining.test.ts new file mode 100644 index 0000000000..27c2012565 --- /dev/null +++ b/backend/src/__tests__/capabilities-mining.test.ts @@ -0,0 +1,98 @@ +import { CORE_READING_MAX_AGE_SECONDS, miningIndexVerdict, type MiningIndexFacts } from '../api/capabilities.mining'; + +/** + * The incident behind these: on 2026-09-23 /api/v1/capabilities called mining + * ready with the index at block 968172 and Core at 968299, because readiness + * was "the tables have rows". Every case below is a way that rule was wrong. + */ + +const NOW = Date.parse('2026-09-23T18:00:00.000Z'); + +function facts(overrides: Partial = {}): MiningIndexFacts { + return { + blockRows: 11_000, + highestHeight: 968_299, + poolRows: 180, + core: { blocks: 968_299, chain: 'main', initialBlockDownload: false, checkedAt: new Date(NOW - 10_000).toISOString() }, + network: 'mainnet', + maxBehindTip: 3, + now: NOW, + ...overrides, + }; +} + +describe('mining index readiness', () => { + it('is ready when the indexed tip matches a fresh same-network Core reading', () => { + expect(miningIndexVerdict(facts())).toEqual({ + state: 'ready', degradedReason: null, indexedTip: 968_299, bitcoinCoreTip: 968_299, lagBlocks: 0, maxLagBlocks: 3, + }); + }); + + it('refuses the production state that was published as ready', () => { + const verdict = miningIndexVerdict(facts({ highestHeight: 968_172 })); + expect(verdict.state).toBe('degraded'); + expect(verdict.lagBlocks).toBe(127); + expect(verdict.degradedReason).toBe('The mining index is 127 blocks behind Bitcoin Core, more than the 3 allowed.'); + }); + + it('holds the lag bound exactly at its boundary', () => { + expect(miningIndexVerdict(facts({ highestHeight: 968_296 })).state).toBe('ready'); + expect(miningIndexVerdict(facts({ highestHeight: 968_295 })).state).toBe('degraded'); + expect(miningIndexVerdict(facts({ highestHeight: 968_299, maxBehindTip: 0 })).state).toBe('ready'); + expect(miningIndexVerdict(facts({ highestHeight: 968_298, maxBehindTip: 0 })).state).toBe('degraded'); + }); + + it('never lets historical rows alone prove readiness', () => { + expect(miningIndexVerdict(facts({ core: null })).state).toBe('unknown'); + }); + + it('reports an empty index and missing pool metadata as degraded before judging currency', () => { + expect(miningIndexVerdict(facts({ blockRows: 0, highestHeight: null }))).toMatchObject({ + state: 'degraded', indexedTip: null, lagBlocks: null, + degradedReason: 'Block indexing is running but no block has been indexed yet.', + }); + expect(miningIndexVerdict(facts({ poolRows: 0 }))).toMatchObject({ + state: 'degraded', degradedReason: 'Mining pool metadata has not been imported yet.', + }); + }); + + it('keeps unknown separate when the Core reading has expired, and publishes no stale lag', () => { + const expired = new Date(NOW - (CORE_READING_MAX_AGE_SECONDS + 1) * 1000).toISOString(); + const verdict = miningIndexVerdict(facts({ core: { blocks: 968_299, chain: 'main', initialBlockDownload: false, checkedAt: expired } })); + expect(verdict).toMatchObject({ state: 'unknown', bitcoinCoreTip: null, lagBlocks: null }); + const edge = new Date(NOW - CORE_READING_MAX_AGE_SECONDS * 1000).toISOString(); + expect(miningIndexVerdict(facts({ core: { blocks: 968_299, chain: 'main', initialBlockDownload: false, checkedAt: edge } })).state).toBe('ready'); + expect(miningIndexVerdict(facts({ core: { blocks: 968_299, chain: 'main', initialBlockDownload: false, checkedAt: 'not a time' } })).state).toBe('unknown'); + }); + + it('refuses a Core reading from another network or one that does not say', () => { + for (const chain of ['test', 'signet', null]) { + const verdict = miningIndexVerdict(facts({ core: { blocks: 968_299, chain, initialBlockDownload: false, checkedAt: new Date(NOW).toISOString() } })); + expect(verdict.state).toBe('unknown'); + expect(verdict.degradedReason).toContain('network'); + } + }); + + it('matches each served network to Core\'s own chain name', () => { + for (const [network, chain] of [['signet', 'signet'], ['testnet', 'test'], ['testnet4', 'testnet4'], ['regtest', 'regtest']]) { + const verdict = miningIndexVerdict(facts({ network, core: { blocks: 968_299, chain, initialBlockDownload: false, checkedAt: new Date(NOW).toISOString() } })); + expect(verdict.state).toBe('ready'); + } + expect(miningIndexVerdict(facts({ network: 'unlisted' })).state).toBe('unknown'); + }); + + it('calls a node still in initial block download syncing, not ready', () => { + const verdict = miningIndexVerdict(facts({ core: { blocks: 968_299, chain: 'main', initialBlockDownload: true, checkedAt: new Date(NOW).toISOString() } })); + expect(verdict.state).toBe('syncing'); + }); + + it('withholds readiness while the index is ahead of Core after a reorganization', () => { + const verdict = miningIndexVerdict(facts({ highestHeight: 968_301 })); + expect(verdict).toMatchObject({ state: 'unknown', lagBlocks: -2 }); + }); + + it('recovers to ready once the index catches up again', () => { + expect(miningIndexVerdict(facts({ highestHeight: 968_100 })).state).toBe('degraded'); + expect(miningIndexVerdict(facts({ highestHeight: 968_298 })).state).toBe('ready'); + }); +}); diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index e88b738f16..42e1bd7a10 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -44,6 +44,7 @@ describe('Mempool Backend Config', () => { POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', POOLS_JSON_FILE: 'tasks/pools/pools-v2.json', POOLS_UPDATE_DELAY: 604800, + MINING_MAX_BEHIND_TIP: 3, AUDIT: false, RUST_GBT: true, LIMIT_GBT: false, diff --git a/backend/src/api/backend-info.ts b/backend/src/api/backend-info.ts index 3d4951bd69..4706878caa 100644 --- a/backend/src/api/backend-info.ts +++ b/backend/src/api/backend-info.ts @@ -78,6 +78,7 @@ class BackendInfo { initialBlockDownload: !!info.initialblockdownload, verificationProgress: info.verificationprogress, checkedAt: new Date().toISOString(), + chain: typeof (info as { chain?: unknown }).chain === 'string' ? String(info.chain) : null, }; } catch (e) { logger.debug(`Could not read chain sync state. Reason: ${(e instanceof Error ? e.message : e)}`); diff --git a/backend/src/api/capabilities.mining.ts b/backend/src/api/capabilities.mining.ts new file mode 100644 index 0000000000..8fc178ae80 --- /dev/null +++ b/backend/src/api/capabilities.mining.ts @@ -0,0 +1,112 @@ +/** + * Whether the mining index is current, judged the only way that means + * anything: its highest indexed block against a fresh Bitcoin Core reading + * of the same network. + * + * The earlier rule called mining ready whenever the blocks and pools tables + * had rows. On 2026-09-23 that published "ready" while the index stood at + * block 968172 and Core at 968299: historical rows proved the index had once + * run, not that it was still running. Time since the newest block was mined + * is not a substitute either, because block intervals vary by hours on their + * own and say nothing about the collector. + * + * Kept free of I/O so every branch is tested directly; capabilities.ts reads + * the tables and the node and hands the facts in. + */ + +/** + * A Core reading older than this cannot vouch for the index. backend-info + * refreshes it every 30 seconds, so four missed refreshes means the node or + * the poller has stopped answering, and "current" can no longer be claimed. + */ +export const CORE_READING_MAX_AGE_SECONDS = 120; + +export type MiningIndexState = 'ready' | 'syncing' | 'degraded' | 'unknown'; + +export interface CoreReading { + readonly blocks: number; + /** Core's own chain name (`main`, `test`, `testnet4`, `signet`, `regtest`), when reported. */ + readonly chain: string | null; + readonly initialBlockDownload: boolean; + readonly checkedAt: string; +} + +export interface MiningIndexFacts { + readonly blockRows: number; + readonly highestHeight: number | null; + readonly poolRows: number; + readonly core: CoreReading | null; + /** The network this backend serves, as configured (`MEMPOOL.NETWORK`). */ + readonly network: string; + readonly maxBehindTip: number; + readonly now: number; +} + +export interface MiningIndexVerdict { + readonly state: MiningIndexState; + readonly degradedReason: string | null; + readonly indexedTip: number | null; + readonly bitcoinCoreTip: number | null; + readonly lagBlocks: number | null; + readonly maxLagBlocks: number; +} + +/** Core's name for each network this backend can be configured to serve. */ +const CORE_CHAIN: Readonly> = { + mainnet: 'main', + '': 'main', + testnet: 'test', + testnet4: 'testnet4', + signet: 'signet', + regtest: 'regtest', + liquid: 'liquidv1', + liquidtestnet: 'liquidtestnet', +}; + +export function miningIndexVerdict(facts: MiningIndexFacts): MiningIndexVerdict { + const indexedTip = facts.blockRows > 0 && Number.isSafeInteger(facts.highestHeight) ? facts.highestHeight : null; + const core = facts.core; + const coreTip = core && Number.isSafeInteger(core.blocks) && core.blocks >= 0 ? core.blocks : null; + const verdict = (state: MiningIndexState, degradedReason: string | null, reference: number | null = coreTip): MiningIndexVerdict => ({ + state, + degradedReason, + indexedTip, + bitcoinCoreTip: reference, + lagBlocks: indexedTip !== null && reference !== null ? reference - indexedTip : null, + maxLagBlocks: facts.maxBehindTip, + }); + + if (facts.blockRows === 0 || indexedTip === null) { + return verdict('degraded', 'Block indexing is running but no block has been indexed yet.'); + } + if (facts.poolRows === 0) { + return verdict('degraded', 'Mining pool metadata has not been imported yet.'); + } + // From here the index has data. Whether it is current needs a reference, + // and a missing, stale or foreign reference is an unknown, not a verdict. + if (!core || coreTip === null) { + return verdict('unknown', 'Bitcoin Core has not reported its height, so the index cannot be compared with it.', null); + } + const checkedAt = Date.parse(core.checkedAt); + if (!Number.isFinite(checkedAt) || (facts.now - checkedAt) / 1000 > CORE_READING_MAX_AGE_SECONDS) { + return verdict('unknown', 'The last Bitcoin Core reading is too old to judge the index against.', null); + } + const expectedChain = CORE_CHAIN[facts.network]; + if (!expectedChain || core.chain !== expectedChain) { + return verdict('unknown', 'Bitcoin Core did not confirm it is on the network this backend serves.', null); + } + if (core.initialBlockDownload) { + return verdict('syncing', 'Bitcoin Core is still in initial block download, so no index built on it is current yet.'); + } + const lag = coreTip - indexedTip; + if (lag < 0) { + // Core reads lower than the index after a reorganization to a shorter + // chain or while Core itself restarts. Neither side can be trusted to be + // right until they agree again. + return verdict('unknown', 'The mining index is ahead of Bitcoin Core, as happens during a reorganization or a node restart; readiness waits until they agree.'); + } + if (lag > facts.maxBehindTip) { + return verdict('degraded', `The mining index is ${lag} blocks behind Bitcoin Core, more than the ${facts.maxBehindTip} allowed.`); + } + return verdict('ready', null); +} diff --git a/backend/src/api/capabilities.routes.ts b/backend/src/api/capabilities.routes.ts index e20c95154f..29aa854404 100644 --- a/backend/src/api/capabilities.routes.ts +++ b/backend/src/api/capabilities.routes.ts @@ -13,50 +13,11 @@ class CapabilitiesRoutes { app.get(config.MEMPOOL.API_URL_PREFIX + 'capabilities', async (req: Request, res: Response) => { try { /* - * IMPLEMENTATION-HANDOFF [M23-HEALTH] | F-M23-03, F-M23-04 - * Coverage: C-BTC-INDEX, C-STATS-FRESH, C-MINING-READY, C-ADDRESS-READY. - * Preparation only, 2026-09-23. Dependencies: M23-BASE, M23-NET. - * Live operational observation, not a mainnet functional test: - * /api/v1/capabilities at 2026-09-23T17:39:41.778Z reported address - * index unavailable, statistics lag 80936 seconds, and mining ready - * despite its indexed height 968172 versus backend-info Core 968299. - * The running backend named 537235052, not the inspected branch tip. - * Verified current-source cause for the readiness gap is in - * capabilities.ts:$miningReport: indexed = total > 0 && poolCount > 0; - * highest/newest are reported but never constrain readiness. Causes - * of the actual address outage and stalled ingestion remain unresolved. - * Sources: Bitcoin Core 31.0 getblockchaininfo RPC; repository - * capabilities.ts, bitcoin/address-index.ts, backend-info-checkpoint.ts; - * user requirements for authoritative freshness and truthful states. - * 1. In capabilities.ts:$miningReport compare the indexed checkpoint - * to a fresh, same-network Core observation and expose the existing - * indexedTip/bitcoinCoreTip/lagBlocks fields. Define a bounded allowed - * lag in the owning configuration contract, tested at its boundary; - * do not infer collector freshness from block mining timestamps alone. - * Preserve unknown separately when a reference checkpoint is absent, - * stale or inconsistent. Existing historical rows must not prove ready. - * 2. Keep this endpoint, release gates and frontend consumers on that - * shared result; coordinate any response/type changes. Exercise cache - * expiry and network identity so a prior ready response cannot conceal - * dependency loss or survive a chain change. Do not substitute HTTP 200 - * or a row count for the final outcome. - * 3. Diagnose the live outage through authorized service/configuration - * identity, cookie freshness, index reachability and ingestion logs. - * Compare Core, completed block cache, SQL tip, address index and - * statistics collector independently. Do not assert one common cause - * without evidence; never restart an active reorg/indexer blindly. - * 4. Add focused tests in the existing capabilities test suite (locate - * before editing): fresh/lagged/unknown tip, database loss, empty rows, - * pool metadata missing, observation expiry, recovery and reorg. - * Run cd backend; npm run test:ci -- --runInBand; npm run lint; - * npm run build. Commands are declared, not executed on SERVER here. - * Acceptance additionally requires real Signet API-to-UI address, - * UTXO/history, mining and statistics outcomes across reload/reconnect; - * faults belong in an isolated environment, not production. Preserve - * historical coverage and accurately report gaps, without fake backfill. - * Rollback: retain the prior artifact, configuration and database backup; - * no destructive rescan, migration or permission relaxation is authorized - * merely by this annotation. Record the verified cause before repair. + * Outstanding: M23-HEALTH of the 2026-09-23 mainnet plan. Mining readiness now + * compares the indexed tip with a fresh same-network Core reading + * (capabilities.mining.ts); the live address and statistics outage causes + * are still being diagnosed. The plan and its acceptance rows live in the + * handoff bundle, not here. */ const report = await capabilities.$report(); res.header('Pragma', 'public'); diff --git a/backend/src/api/capabilities.ts b/backend/src/api/capabilities.ts index 18b1e96f81..c19482915c 100644 --- a/backend/src/api/capabilities.ts +++ b/backend/src/api/capabilities.ts @@ -5,6 +5,7 @@ import backendInfo from './backend-info'; import { Common } from './common'; import { preflightFailures, type PreflightFailure, type PreflightInput } from './capabilities.preflight'; import { $optionalCapabilityReports } from './capabilities.optional'; +import { miningIndexVerdict } from './capabilities.mining'; import { $probeAddressIndex, addressBackendKind, @@ -29,8 +30,13 @@ export type { PreflightFailure, PreflightInput }; * not finished yet is not the same as one that is broken, and a reader who is * told "unavailable" about an index that will answer in an hour has been told * the wrong thing. + * + * `unknown` exists because a verdict needs a reference. When the reading a + * feature is judged against is missing, too old or from another network, the + * honest report is that its currency cannot be established, not that it is + * ready because its tables have rows. */ -export type CapabilityState = 'ready' | 'syncing' | 'degraded' | 'unavailable' | 'disabled'; +export type CapabilityState = 'ready' | 'syncing' | 'degraded' | 'unavailable' | 'disabled' | 'unknown'; export interface CapabilityDependency { readonly name: string; @@ -435,22 +441,41 @@ class Capabilities { detail: poolCount > 0 ? null : 'No mining pool metadata has been imported.', }); - const indexed = total > 0 && poolCount > 0; + // Rows prove the index ran once, not that it is running. Readiness is + // the highest indexed block against a fresh Core reading of the same + // network; see capabilities.mining.ts for every branch. + const chainSync = backendInfo.getBackendInfo().chainSync; + const verdict = miningIndexVerdict({ + blockRows: total, + highestHeight: highest, + poolRows: poolCount, + core: chainSync ? { + blocks: chainSync.blocks, + chain: chainSync.chain ?? null, + initialBlockDownload: chainSync.initialBlockDownload, + checkedAt: chainSync.checkedAt, + } : null, + network: config.MEMPOOL.NETWORK, + maxBehindTip: config.MEMPOOL.MINING_MAX_BEHIND_TIP, + now: Date.now(), + }); return { enabled, routesRegistered, dependencies, - state: indexed ? 'ready' : 'degraded', + state: verdict.state, coverage: { from: lowest === null ? null : String(lowest), to: highest === null ? null : String(highest), }, rowCount: total, lastSuccessfulUpdate: newest ? newest.toISOString() : null, + // Age of the newest indexed block's own timestamp. Reported for + // context only: block intervals vary by hours, so it decides nothing. lagSeconds: newest ? Math.max(0, Math.round((Date.now() - newest.getTime()) / 1000)) : null, - degradedReason: total === 0 - ? 'Block indexing is running but no block has been indexed yet.' - : poolCount === 0 - ? 'Mining pool metadata has not been imported yet.' - : null, + degradedReason: verdict.degradedReason, + indexedTip: verdict.indexedTip, + bitcoinCoreTip: verdict.bitcoinCoreTip, + lagBlocks: verdict.lagBlocks, + maxLagBlocks: verdict.maxLagBlocks, }; } catch (e) { logger.debug('Capability probe could not read the mining index: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/config.ts b/backend/src/config.ts index 87d6091228..1cc2f6c1a9 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -35,6 +35,7 @@ interface IConfig { POOLS_JSON_TREE_URL: string, POOLS_JSON_FILE: string, POOLS_UPDATE_DELAY: number, + MINING_MAX_BEHIND_TIP: number, AUDIT: boolean; RUST_GBT: boolean; LIMIT_GBT: boolean; @@ -211,6 +212,7 @@ const defaults: IConfig = { 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'POOLS_JSON_FILE': 'tasks/pools/pools-v2.json', 'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week + 'MINING_MAX_BEHIND_TIP': 3, // blocks the mining index may trail Core and still be ready 'AUDIT': false, 'RUST_GBT': true, 'LIMIT_GBT': false, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 2a8a89f33e..90e6d84b6e 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -538,6 +538,11 @@ export interface IChainSyncState { /** Fraction of the chain verified, 0 to 1, as the node reports it. */ verificationProgress: number; checkedAt: string; + /** + * The chain Core says it is on (main, test, testnet4, signet, regtest), so a + * reading can be matched to the network it is compared against. + */ + chain?: string | null; } export interface INetworkInfo { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 6298029745..42f2b3db25 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -39,6 +39,7 @@ "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "POOLS_JSON_FILE": "__MEMPOOL_POOLS_JSON_FILE__", "POOLS_UPDATE_DELAY": __MEMPOOL_POOLS_UPDATE_DELAY__, + "MINING_MAX_BEHIND_TIP": __MEMPOOL_MINING_MAX_BEHIND_TIP__, "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index f996626a6c..2cf9a15eb0 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -34,6 +34,7 @@ __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=""} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=""} __MEMPOOL_POOLS_JSON_FILE__=${MEMPOOL_POOLS_JSON_FILE:=tasks/pools/pools-v2.json} __MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800} +__MEMPOOL_MINING_MAX_BEHIND_TIP__=${MEMPOOL_MINING_MAX_BEHIND_TIP:=3} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true} __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false} @@ -264,6 +265,7 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_FILE__!${__MEMPOOL_POOLS_JSON_FILE__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_MINING_MAX_BEHIND_TIP__!${__MEMPOOL_MINING_MAX_BEHIND_TIP__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json diff --git a/docs/operations/CONFIGURATION.md b/docs/operations/CONFIGURATION.md index f91b934b8b..2a037257e4 100644 --- a/docs/operations/CONFIGURATION.md +++ b/docs/operations/CONFIGURATION.md @@ -47,6 +47,7 @@ the default from `config.ts`. | `CPFP_INDEXING` | `false` | Index child-pays-for-parent clusters | | `AUDIT` | `false` | Block template auditing, which compares mined blocks against what the node expected | | `RUST_GBT` | `true` | Use the Rust block template builder in `rust/gbt`. Turning it off falls back to the TypeScript implementation | +| `MINING_MAX_BEHIND_TIP` | `3` | How many blocks the mining index may trail a fresh Core reading of the same network and still be reported `ready` in `/api/v1/capabilities`. Beyond it mining reads `degraded`; a missing, stale (over 120 s) or other-network Core reading reads `unknown` | | `AUTOMATIC_POOLS_UPDATE` | `false` | **Leave off.** Turning it on makes the backend fetch mining pool metadata over the network at runtime, from `POOLS_JSON_URL`. The bundled `backend/src/tasks/pools/pools-v2.json` is used instead | | `MAX_PUSH_TX_SIZE_WEIGHT` | `400000` | Largest transaction the broadcast route accepts, in weight units | | `MAX_TRACKED_ADDRESSES` | `1` | How many addresses one WebSocket client may subscribe to | @@ -254,11 +255,19 @@ defaults. `UNIVERSE_CHAIN_NETWORKS` (Docker frontend: `UNIVERSE_CHAIN_NETWORKS`, a JSON string) names which network each non-Bitcoin chain is read from, for example -`{"dogecoin":"testnet"}`. Values are `mainnet`, `testnet` or `regtest` per -chain; the default `{}` reads every chain from mainnet. Bitcoin is never listed: -it follows the network selector, and the selector never implies a Dogecoin or -Zcash network. An entry the frontend cannot use is ignored with a console -warning and that chain reads mainnet. The overlay must serve the named scope +`{"dogecoin":"testnet"}`. Keys are `dogecoin`, `zcash` and `fractal`; values are +the networks the overlay serves for that chain (`mainnet`, `testnet` or `regtest` +for Dogecoin and Zcash, `mainnet` or `testnet` for Fractal). The default `{}`, +and any chain the map does not name, reads mainnet. Bitcoin is never listed: it +follows the network selector, and the selector never implies a Dogecoin or +Zcash network. + +A value that is present but wrong never falls back to mainnet. Unreadable JSON, +a non-object, a Bitcoin entry or an unknown key (a typo such as `doge`) makes +every listed chain unavailable; an unsupported network makes that one chain +unavailable. An unavailable chain sends no request and opens no live socket, +its pages say the setting is invalid and why, and the chain picker names it +"Network setting invalid". Correct the value and reload. The overlay must serve the named scope (`UNIVERSE_DOGECOIN_NETWORKS` for Dogecoin, the indexer's declared network for Zcash); a scope it does not serve is shown as unavailable under that network, never as mainnet data. From 663498db7a82c8f90b3ff3c2bfa7e6fc54b56018 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 20:57:02 +0000 Subject: [PATCH 07/11] fix(release): carry the acceptance evidence closure in the artifact and qualify the packed archive M23-PACK, F-M23-01. The artifact workflow staged docs/ but left it out of the tar member list, and never staged the evidence files the acceptance envelope names, so a candidate qualified in the checkout could not qualify on the host. protocol-contract.mjs --stage-acceptance copies the manifest, envelope and complete evidence closure under docs/, checking every file with the gate's own rules (plain docs/ path, inside the root, not a symlink, recorded SHA-256) before and after the copy. docs is now packed, and qualify-artifact.mjs lists the archive (refusing traversal, absolute names and links in the evidence tree), extracts it into an empty directory and runs the gate the archive carries, before upload. release-artifact.test.mjs proves it on real gzip tar archives: a valid Signet-qualified Mainnet candidate, missing docs, missing nested evidence, tampered bytes, a symlink member, traversal names and stale revisions. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/universe-ci.yml | 2 +- .../workflows/universe-release-artifact.yml | 66 ++-- scripts/universe/protocol-contract.mjs | 132 ++++++- scripts/universe/qualify-artifact.mjs | 156 +++++++++ scripts/universe/release-artifact.test.mjs | 326 ++++++++++++++++++ scripts/universe/release-gates.test.mjs | 4 +- 6 files changed, 641 insertions(+), 45 deletions(-) create mode 100644 scripts/universe/qualify-artifact.mjs create mode 100644 scripts/universe/release-artifact.test.mjs diff --git a/.github/workflows/universe-ci.yml b/.github/workflows/universe-ci.yml index 4063f68837..4d23a0d3f7 100644 --- a/.github/workflows/universe-ci.yml +++ b/.github/workflows/universe-ci.yml @@ -132,7 +132,7 @@ jobs: # the allowlist, green the whole time. These run the script's own text # against healthy and unhealthy fixtures. - name: The release gates fail on the faults they exist for - run: node --test scripts/universe/release-gates.test.mjs + run: node --test scripts/universe/release-gates.test.mjs scripts/universe/release-artifact.test.mjs - name: Workflow inputs remain data and builds preserve the runner host run: node --test scripts/universe/workflow-safety.test.mjs scripts/universe/backend-startup-check.test.mjs diff --git a/.github/workflows/universe-release-artifact.yml b/.github/workflows/universe-release-artifact.yml index 53717d096f..c5c99a5e23 100644 --- a/.github/workflows/universe-release-artifact.yml +++ b/.github/workflows/universe-release-artifact.yml @@ -120,43 +120,12 @@ jobs: # compare it against the running release and refuse the hard link when it # differs. That check belongs there, where both trees are visible; here # only one of them is. - # IMPLEMENTATION-HANDOFF [M23-PACK] | F-M23-01 | C-RELEASE-ARCHIVE - # Preparation only, 2026-09-23. Prerequisites: M23-BASE and the existing - # qualified acceptance work; this is not permission to bypass that gate. - # Verified at 079dc0d79755bc986bfae3288ffe0e479da3e1f6: this step copies - # two files into stage/docs, but tar below names only backend/frontend/ - # scripts/production/RELEASE-MANIFEST.json. docs is therefore absent. - # release.sh:gate_qualified_acceptance requires the candidate's protocol - # manifest and rooted evidence before cutover. A successful checkout-side - # qualification does not make the extracted archive self-contained. - # Governing sources: scripts/universe/release.sh:gate_qualified_acceptance; - # protocol-contract.mjs:verifyEvidence; user release/evidence requirements. - # 1. Stage the protocol manifest, acceptance envelope and every evidence - # file referenced by that envelope under a candidate-contained root. - # Enumerate and validate the exact reference closure using the existing - # verifier's schema/path rules. Reject missing files, hash mismatches, - # absolute/traversal paths and symlinks escaping the candidate. Do not - # copy credentials, whole workspaces, or unsupported success assertions. - # 2. Include those staged paths in the archive member list. Keep the - # production artifact and evidence bound to the accepted full source SHA - # and production configuration; do not use a mutable external evidence root. - # 3. Extract the actual archive into a new directory, with the checkout - # unavailable, and run the same protocol-contract --release invocation - # with --acceptance-root set to the extracted candidate. Assert all - # referenced evidence members exist with their expected hashes before - # upload. Keep all existing branding, identity and release gates. - # 4. Extend scripts/universe/release-gates.test.mjs with actual tar member - # and extracted-candidate qualification cases, not regex presence alone. - # Cover missing docs, missing nested evidence, tampered bytes, escaped - # symlinks, stale revision and a valid Signet-qualified mainnet envelope. - # Commands declared, not executed on SERVER in this preparation: - # node --test scripts/universe/release-gates.test.mjs - # node --test scripts/universe/protocol-contract.test.mjs - # bash -n scripts/universe/release.sh - # Acceptance: the exact uploaded archive qualifies after extraction with - # no checkout dependency; failed qualification prevents publication/cutover. - # Rollback: retain the previous immutable artifact, routing and compatible - # configuration. No database migration or live transaction is needed here. + # The artifact carries its own acceptance: the protocol manifest, the + # acceptance envelope and every evidence file the envelope names, staged + # under docs/ with the release gate's own path and digest rules, and + # docs is in the member list. Before 2026-09-23 docs/ was staged but not + # packed, so a candidate that qualified here could never qualify on the + # host. The step after this one proves it on the packed archive. - name: Pack id: pack run: | @@ -180,9 +149,10 @@ jobs: cp -a backend/rust-gbt "$stage/backend/rust-gbt" cp -a frontend/dist/mempool/browser "$stage/frontend/build" cp -a scripts/universe "$stage/scripts/universe" - mkdir -p "$stage/docs/protocols" "$stage/docs/acceptance" - cp -a docs/protocols/PROTOCOL-COVERAGE.json "$stage/docs/protocols/PROTOCOL-COVERAGE.json" - cp -a docs/acceptance/qualified-release-evidence.json "$stage/docs/acceptance/qualified-release-evidence.json" + node scripts/universe/protocol-contract.mjs --stage-acceptance "$stage" \ + --manifest docs/protocols/PROTOCOL-COVERAGE.json \ + --acceptance docs/acceptance/qualified-release-evidence.json \ + --acceptance-root "$GITHUB_WORKSPACE" # The unit files travel with the release rather than being fetched # from a checkout that happens to be on the right commit. Adopting # socket activation needed both of these on the host, and taking @@ -205,10 +175,24 @@ jobs: test -f "$stage/frontend/build/index.html" test -f "$stage/scripts/universe/gateway.mjs" out="$PWD/mempool-$sha.tar.gz" - tar -czf "$out" -C "$stage" backend frontend scripts production RELEASE-MANIFEST.json + tar -czf "$out" -C "$stage" backend frontend scripts production docs RELEASE-MANIFEST.json sha256sum "$out" | tee "$out.sha256" printf 'name=mempool-%s\n' "$sha" >> "$GITHUB_OUTPUT" + # The exact archive about to be uploaded, extracted into an empty + # directory and qualified by the gate it carries, with that directory + # as the evidence root. Nothing from the checkout is read, so an archive + # that needs the checkout to pass fails here and is never published. + - name: Qualify the packed archive on its own + run: | + set -euo pipefail + archive="$PWD/mempool-${{ steps.sha.outputs.short }}.tar.gz" + work=$(mktemp -d) + cp "$archive" "$work/" + cd "$work" + node "$GITHUB_WORKSPACE/scripts/universe/qualify-artifact.mjs" "$work/$(basename "$archive")" \ + --commit '${{ steps.sha.outputs.sha }}' --network mainnet + - name: Upload uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: diff --git a/scripts/universe/protocol-contract.mjs b/scripts/universe/protocol-contract.mjs index 3797c0e8c8..9f6ec7a960 100644 --- a/scripts/universe/protocol-contract.mjs +++ b/scripts/universe/protocol-contract.mjs @@ -36,7 +36,9 @@ import { createHash } from 'node:crypto'; import { + copyFileSync, lstatSync, + mkdirSync, readFileSync, realpathSync, } from 'node:fs'; @@ -812,6 +814,112 @@ function verifyEvidenceFiles(entries, context, report, owner) { } } +// --------------------------------------------------------------------------- +// The evidence a release artifact carries +// --------------------------------------------------------------------------- + +/** Where a release artifact carries its protocol manifest and acceptance envelope. */ +export const STAGED_MANIFEST_PATH = 'docs/protocols/PROTOCOL-COVERAGE.json'; +export const STAGED_ACCEPTANCE_PATH = 'docs/acceptance/qualified-release-evidence.json'; + +/** + * Every evidence file an acceptance envelope names: the configuration proof's + * and every row's. This is the set an artifact must carry for the release gate + * to qualify it with nothing but the artifact. Repeated references to one + * path are kept once; a path named with two different digests is kept twice, + * so the digest check refuses one of them rather than one silently winning. + */ +export function acceptanceEvidenceClosure(evidence) { + const lists = [evidence?.candidate?.configurationProof?.evidence]; + for (const row of Array.isArray(evidence?.rows) ? evidence.rows : []) { + lists.push(row?.evidence); + } + const seen = new Set(); + const closure = []; + for (const list of lists) { + if (!Array.isArray(list)) continue; + for (const entry of list) { + const key = JSON.stringify([entry?.path, typeof entry?.sha256 === 'string' ? entry.sha256.toLowerCase() : entry?.sha256]); + if (seen.has(key)) continue; + seen.add(key); + closure.push(entry); + } + } + return closure; +} + +function regularFile(file, label, report) { + try { + const stat = lstatSync(file); + if (stat.isSymbolicLink()) { + report.fail(`The ${label} ${file} is a symlink, not a file.`); + return false; + } + if (!stat.isFile()) { + report.fail(`The ${label} ${file} is not a file.`); + return false; + } + return true; + } catch (error) { + report.fail(`The ${label} ${file} could not be read: ${error instanceof Error ? error.message : error}.`); + return false; + } +} + +/** + * Copies the protocol manifest, the acceptance envelope and the complete + * evidence closure the envelope names into a release staging directory. + * + * The artifact used to carry the two documents without the files the + * envelope points at, and then not even the documents: the workflow staged + * docs/ but left it out of the archive, so a candidate qualified in the + * checkout could never qualify on the host. Every evidence file is checked + * with the release gate's own rules before and after the copy: a relative + * path under docs/, inside the evidence root, a regular file rather than a + * symlink, and the recorded SHA-256. Nothing is copied when any check fails. + */ +export function stageAcceptance({ manifestPath, acceptancePath, acceptanceRoot, stageRoot }, report = new Report()) { + const manifestOk = regularFile(manifestPath, 'protocol manifest', report); + const envelopeOk = regularFile(acceptancePath, 'acceptance envelope', report); + if (!manifestOk || !envelopeOk) return report; + + let evidence; + try { + evidence = JSON.parse(readFileSync(acceptancePath, 'utf8')); + } catch (error) { + report.fail(`The acceptance envelope is not readable JSON: ${error instanceof Error ? error.message : error}.`); + return report; + } + const closure = acceptanceEvidenceClosure(evidence); + if (!closure.length) { + report.fail('The acceptance envelope names no evidence files to carry.'); + return report; + } + for (const entry of closure) { + const name = typeof entry?.path === 'string' ? entry.path.replaceAll('\\', '/') : ''; + if (!name.startsWith('docs/') || path.posix.normalize(name) !== name) { + report.fail(`Evidence ${JSON.stringify(entry?.path)} is not a plain path under docs/, where the artifact carries evidence.`); + } + } + const source = evidenceRoot(acceptanceRoot, report); + verifyEvidenceFiles(closure, source, report, 'The acceptance closure'); + if (report.problems.length) return report; + + const place = (from, relative) => { + const target = path.join(stageRoot, relative); + mkdirSync(path.dirname(target), { recursive: true }); + copyFileSync(from, target); + }; + place(manifestPath, STAGED_MANIFEST_PATH); + place(acceptancePath, STAGED_ACCEPTANCE_PATH); + for (const entry of closure) { + place(realpathSync(path.resolve(source.rootPath, entry.path)), entry.path); + } + // Staged copies are what ship, so they are what get checked last. + verifyEvidenceFiles(closure, evidenceRoot(stageRoot, report), report, 'The staged acceptance closure'); + return report; +} + function validateQualifiedAcceptanceEvidence( manifest, descriptors, @@ -1429,7 +1537,9 @@ function usage(message) { ' protocol-contract.mjs --check the offline gate\n' + ' protocol-contract.mjs --against compare the pin against what is served\n' + ' protocol-contract.mjs --release the release gate, stricter than --check\n' + - ' [--expect-sha ] [--expect-artifact-commit ] [--network ] [--acceptance ] [--acceptance-root ]\n', + ' [--expect-sha ] [--expect-artifact-commit ] [--network ] [--acceptance ] [--acceptance-root ]\n' + + ' protocol-contract.mjs --stage-acceptance copy the manifest, envelope and evidence closure into a release stage\n' + + ' --manifest --acceptance --acceptance-root \n', ); process.exit(2); } @@ -1446,15 +1556,33 @@ async function main() { const networkIndex = argv.indexOf('--network'); const acceptanceIndex = argv.indexOf('--acceptance'); const acceptanceRootIndex = argv.indexOf('--acceptance-root'); + const stageIndex = argv.indexOf('--stage-acceptance'); + const manifestIndex = argv.indexOf('--manifest'); const modes = [ wantsRecord, wantsCheck, againstIndex !== -1, releaseIndex !== -1, + stageIndex !== -1, ].filter(Boolean); if (modes.length !== 1) { - usage('Pass exactly one of --record, --check, --against, --release.'); + usage('Pass exactly one of --record, --check, --against, --release, --stage-acceptance.'); + } + if (stageIndex !== -1) { + const value = (index, flag) => { + if (index === -1 || !argv[index + 1]) usage(`--stage-acceptance needs ${flag}.`); + return argv[index + 1]; + }; + const stageRoot = value(stageIndex, 'a staging directory'); + stageAcceptance({ + manifestPath: value(manifestIndex, '--manifest '), + acceptancePath: value(acceptanceIndex, '--acceptance '), + acceptanceRoot: value(acceptanceRootIndex, '--acceptance-root '), + stageRoot, + }).throwIfFailed('The acceptance evidence could not be staged for the release artifact.'); + process.stdout.write(`Staged the protocol manifest, acceptance envelope and evidence closure into ${stageRoot}.\n`); + return; } if (releaseIndex !== -1) { if (!argv[releaseIndex + 1]) usage('--release needs a url or file.'); diff --git a/scripts/universe/qualify-artifact.mjs b/scripts/universe/qualify-artifact.mjs new file mode 100644 index 0000000000..a4cd2e0c49 --- /dev/null +++ b/scripts/universe/qualify-artifact.mjs @@ -0,0 +1,156 @@ +#!/usr/bin/env node +/** + * Qualifies a packed release artifact with nothing but the artifact. + * + * node scripts/universe/qualify-artifact.mjs .tar.gz> --commit [--network mainnet] + * + * The artifact workflow used to qualify the acceptance envelope in the + * checkout and then pack an archive without it: docs/ was staged and left out + * of the tar member list, and the evidence files the envelope names were never + * staged at all. release.sh then refused the candidate on the host, or a + * candidate that passed in the checkout would have needed the checkout to + * pass again. This reads the archive the way the host will: + * + * 1. Lists its members and refuses absolute or parent-relative names, and any + * link under docs/, before extracting a byte. + * 2. Requires the release manifest, the protocol manifest, the acceptance + * envelope and the gate script inside the archive. + * 3. Extracts into a fresh directory, holds RELEASE-MANIFEST.json to the + * commit being released, and runs the release gate from the extracted + * protocol-contract.mjs with the extracted directory as the evidence root, + * so every referenced evidence file is checked for presence, containment + * and SHA-256 from inside the artifact. + * + * Exit 0 only when the exact archive qualifies. The workflow runs this before + * upload, so an archive that fails here is never published. + */ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const REQUIRED_MEMBERS = [ + 'RELEASE-MANIFEST.json', + 'docs/protocols/PROTOCOL-COVERAGE.json', + 'docs/acceptance/qualified-release-evidence.json', + 'scripts/universe/protocol-contract.mjs', +]; + +/** + * Runs tar from the archive's own directory with a bare file name, because + * GNU tar reads a drive-letter path such as C:\x as a remote host. + */ +function tar(args, archive) { + const result = spawnSync('tar', [...args, '-f', path.basename(archive)], { + cwd: path.dirname(archive), encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, + }); + if (result.error) throw new Error(`tar could not run: ${result.error.message}`); + if (result.status !== 0) throw new Error(`tar ${args[0]} failed: ${(result.stderr || '').trim()}`); + return result.stdout; +} + +function memberName(line) { + return line.replace(/^\.\//, '').replace(/\/$/, ''); +} + +/** Problems with the member list alone, before anything is extracted. */ +export function memberProblems(names, verboseLines) { + const problems = []; + for (const raw of names) { + const name = raw.replace(/^\.\//, ''); + if (!name) continue; + if (name.startsWith('/') || /^[A-Za-z]:/.test(name) || name.split('/').includes('..')) { + problems.push(`The archive member ${JSON.stringify(raw)} is absolute or escapes the release directory.`); + } + } + // A link can point anywhere once extracted, so the evidence tree carries none. + for (const line of verboseLines) { + if (!/^[lh]/.test(line)) continue; + if (/\s(\.\/)?docs\//.test(line)) { + problems.push(`The archive carries a link in its evidence tree: ${line.trim()}`); + } + } + const present = new Set(names.map(memberName)); + for (const required of REQUIRED_MEMBERS) { + if (!present.has(required)) { + problems.push(`The archive does not carry ${required}.`); + } + } + return problems; +} + +/** + * @returns {Promise} problems; empty when the archive qualifies. + */ +export async function qualifyArtifact(archive, { commit, network = 'mainnet' }) { + if (!commit || !/^[0-9a-f]{40}$/i.test(commit)) { + return ['A full 40 character release commit is required.']; + } + const names = tar(['-tz'], archive).split('\n').filter(Boolean); + const verbose = tar(['-tvz'], archive).split('\n').filter(Boolean); + const listed = memberProblems(names, verbose); + if (listed.length) return listed; + + const extracted = mkdtempSync(path.join(tmpdir(), 'qualify-artifact-')); + try { + tar(['-xz', '-C', extracted], archive); + const problems = []; + let manifestCommit; + try { + manifestCommit = JSON.parse(readFileSync(path.join(extracted, 'RELEASE-MANIFEST.json'), 'utf8')).commit; + } catch (error) { + return [`RELEASE-MANIFEST.json is not readable: ${error instanceof Error ? error.message : error}.`]; + } + if (manifestCommit !== commit) { + problems.push(`RELEASE-MANIFEST.json names ${JSON.stringify(manifestCommit)}, not the release commit ${commit}.`); + } + // The gate that runs is the one the artifact carries, not the checkout's. + const contract = await import(pathToFileURL(path.join(extracted, 'scripts', 'universe', 'protocol-contract.mjs')).href); + let manifest; + let acceptanceEvidence; + try { + manifest = JSON.parse(readFileSync(path.join(extracted, contract.STAGED_MANIFEST_PATH ?? 'docs/protocols/PROTOCOL-COVERAGE.json'), 'utf8')); + acceptanceEvidence = JSON.parse(readFileSync(path.join(extracted, contract.STAGED_ACCEPTANCE_PATH ?? 'docs/acceptance/qualified-release-evidence.json'), 'utf8')); + } catch (error) { + return [...problems, `The carried manifest or acceptance envelope is not readable JSON: ${error instanceof Error ? error.message : error}.`]; + } + const report = contract.releaseGate(manifest, { + artifactCommit: commit, + network, + acceptanceEvidence, + acceptanceRoot: extracted, + }); + return [...problems, ...report.problems]; + } finally { + rmSync(extracted, { recursive: true, force: true }); + } +} + +async function main() { + const argv = process.argv.slice(2); + const archive = argv[0]; + const flag = (name) => { + const index = argv.indexOf(name); + return index === -1 ? undefined : argv[index + 1]; + }; + if (!archive || archive.startsWith('--')) { + process.stderr.write('Usage: qualify-artifact.mjs --commit [--network ]\n'); + process.exitCode = 2; + return; + } + const problems = await qualifyArtifact(path.resolve(archive), { commit: flag('--commit'), network: flag('--network') ?? 'mainnet' }); + if (problems.length) { + process.stderr.write(`${archive} does not qualify on its own:\n${problems.map((p) => ` - ${p}`).join('\n')}\n`); + process.exitCode = 1; + return; + } + process.stdout.write(`${archive} qualifies on its own for ${flag('--commit')}.\n`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : error}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/universe/release-artifact.test.mjs b/scripts/universe/release-artifact.test.mjs new file mode 100644 index 0000000000..7712c1c64b --- /dev/null +++ b/scripts/universe/release-artifact.test.mjs @@ -0,0 +1,326 @@ +/** + * The release artifact carries its own acceptance, proven on real archives. + * + * Until 2026-09-23 the artifact workflow staged docs/ and then left it out of + * the tar member list, and never staged the evidence files the acceptance + * envelope names. A candidate that qualified in the checkout could not + * qualify on the host. Regex checks on the workflow text did not notice, + * because the text contained every expected string. These cases build actual + * gzip tar archives, extract them into empty directories and run the gate the + * archive carries, with no checkout involved. + */ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { + copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; +import test from 'node:test'; +import { stageAcceptance, acceptanceEvidenceClosure } from './protocol-contract.mjs'; +import { qualifyArtifact, memberProblems } from './qualify-artifact.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = join(here, '..', '..'); +const workdir = mkdtempSync(join(tmpdir(), 'release-artifact-')); +test.after(() => rmSync(workdir, { recursive: true, force: true })); + +const ARTIFACT_COMMIT = 'd'.repeat(40); +const SOURCE_SHA = 'a'.repeat(40); +const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +// --------------------------------------------------------------------------- +// A minimal ustar writer, so an archive can hold members the host filesystem +// cannot create (a symlink on an unprivileged Windows account, a traversal +// name) and so no case depends on the packing tool it is testing. +// --------------------------------------------------------------------------- + +function header(name, { size = 0, type = '0', linkname = '', mode = 0o644 } = {}) { + const block = Buffer.alloc(512, 0); + const put = (text, offset, length) => block.write(text, offset, Math.min(Buffer.byteLength(text), length), 'utf8'); + const octal = (value, length) => value.toString(8).padStart(length - 1, '0') + '\0'; + put(name, 0, 100); + put(octal(mode, 8), 100, 8); + put(octal(0, 8), 108, 8); + put(octal(0, 8), 116, 8); + put(octal(size, 12), 124, 12); + put(octal(1_758_000_000, 12), 136, 12); + put(' ', 148, 8); + put(type, 156, 1); + put(linkname, 157, 100); + put('ustar\0', 257, 6); + put('00', 263, 2); + put('root', 265, 32); + put('root', 297, 32); + let sum = 0; + for (const byte of block) sum += byte; + put(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8); + return block; +} + +/** entries: [name, Buffer | { symlink: target }] */ +function archive(file, entries) { + const parts = []; + const directories = new Set(); + for (const [name] of entries) { + const segments = name.split('/'); + for (let i = 1; i < segments.length; i += 1) { + const directory = segments.slice(0, i).join('/') + '/'; + if (!directories.has(directory) && !directory.startsWith('..') && !directory.startsWith('/')) { + directories.add(directory); + parts.push(header(directory, { type: '5', mode: 0o755 })); + } + } + } + for (const [name, content] of entries) { + if (Buffer.isBuffer(content)) { + parts.push(header(name, { size: content.length })); + parts.push(content); + const padding = (512 - (content.length % 512)) % 512; + if (padding) parts.push(Buffer.alloc(padding, 0)); + } else { + parts.push(header(name, { type: '2', linkname: content.symlink, mode: 0o777 })); + } + } + parts.push(Buffer.alloc(1024, 0)); + writeFileSync(file, gzipSync(Buffer.concat(parts))); + return file; +} + +// --------------------------------------------------------------------------- +// A candidate whose every declared operation passed on Signet, with a +// separate Mainnet configuration proof, as the release contract requires. +// --------------------------------------------------------------------------- + +function signetQualifiedCandidate() { + const root = mkdtempSync(join(workdir, 'checkout-')); + const pinned = JSON.parse(readFileSync(join(repositoryRoot, 'docs', 'protocols', 'PROTOCOL-COVERAGE.json'), 'utf8')); + const dependencyRevision = 'fixture-backend-dependencies-v1'; + const configurationDigest = 'b'.repeat(64); + const specificationRevision = 'fixture-protocol-spec-v1'; + + const journey = Buffer.from(JSON.stringify({ run: 'signet-fixture', journeys: 'every declared read' })); + const configuration = Buffer.from(JSON.stringify({ network: 'mainnet', endpoints: 'first-party' })); + const journeyPath = 'docs/acceptance/evidence/signet/run-1/journeys.json'; + const configurationPath = 'docs/acceptance/evidence/mainnet/configuration.json'; + for (const [relative, bytes] of [[journeyPath, journey], [configurationPath, configuration]]) { + mkdirSync(dirname(join(root, relative)), { recursive: true }); + writeFileSync(join(root, relative), bytes); + } + + const protocols = pinned.protocols.map((protocol) => ({ + ...protocol, + networks: ['mainnet'], + coverage: 'unknown', + releaseStatus: 'BLOCKED', + readOperationDescriptors: (protocol.readOperationDescriptors ?? []).map((operation) => ({ ...operation, acceptance: 'PASS' })), + })); + const declared = protocols.reduce((total, protocol) => total + protocol.readOperationDescriptors.length, 0); + const manifest = { + ...pinned, + sourceSha: SOURCE_SHA, + protocols, + acceptance: { declared, passed: declared, failed: 0, blocked: 0, notApplicable: 0, notTested: 0, rejected: 0 }, + }; + const envelope = { + schemaVersion: 'universe-explorer-acceptance-v1', + generatedAt: '2026-09-23T20:00:00.000Z', + provenance: { + producer: 'universe-acceptance-runner-v1', + executionEnvironment: 'controlled-offline-fixture', + command: 'fixture acceptance command', + }, + candidate: { + sourceSha: SOURCE_SHA, + artifactCommit: ARTIFACT_COMMIT, + dependencyRevision, + configurationDigest, + specificationRevisions: [specificationRevision], + acceptanceNetwork: 'signet', + deploymentNetwork: 'mainnet', + configurationProof: { + network: 'mainnet', + configurationDigest, + sourceRevision: ARTIFACT_COMMIT, + assertions: ['mainnet endpoints, credentials scope and schemas checked offline'], + evidence: [{ path: configurationPath, sha256: sha256(configuration) }], + }, + }, + rows: protocols.flatMap((protocol) => protocol.readOperationDescriptors.map((operation) => ({ + protocol: protocol.id, + operation: operation.id, + variant: 'default', + role: 'read', + chain: protocol.chain, + network: 'signet', + result: 'PASS', + codeRevision: SOURCE_SHA, + dependencyRevision, + configurationDigest, + specificationRevision, + ranAt: '2026-09-23T19:00:00.000Z', + checkpoint: { height: 1, blockHash: 'c'.repeat(64) }, + evidence: [{ path: journeyPath, sha256: sha256(journey) }], + assertions: ['fixture evidence is bound to the qualified row'], + authorityReadback: ['fixture authority readback is present'], + consumerAssertions: ['fixture consumer assertion is present'], + }))), + exclusions: [], + }; + mkdirSync(join(root, 'docs', 'protocols'), { recursive: true }); + const manifestPath = join(root, 'docs', 'protocols', 'PROTOCOL-COVERAGE.json'); + const acceptancePath = join(root, 'docs', 'acceptance', 'qualified-release-evidence.json'); + writeFileSync(manifestPath, JSON.stringify(manifest)); + writeFileSync(acceptancePath, JSON.stringify(envelope)); + return { root, manifestPath, acceptancePath, envelope, journeyPath, configurationPath }; +} + +/** Stages a candidate the way the workflow's Pack step does, minus the build outputs. */ +function stagedRelease(candidate, { commit = ARTIFACT_COMMIT } = {}) { + const stage = mkdtempSync(join(workdir, 'stage-')); + const report = stageAcceptance({ + manifestPath: candidate.manifestPath, + acceptancePath: candidate.acceptancePath, + acceptanceRoot: candidate.root, + stageRoot: stage, + }); + assert.deepEqual(report.problems, []); + mkdirSync(join(stage, 'scripts', 'universe'), { recursive: true }); + copyFileSync(join(here, 'protocol-contract.mjs'), join(stage, 'scripts', 'universe', 'protocol-contract.mjs')); + writeFileSync(join(stage, 'RELEASE-MANIFEST.json'), JSON.stringify({ commit })); + return stage; +} + +function entriesOf(stage, relatives) { + return relatives.map((relative) => [relative, readFileSync(join(stage, relative))]); +} + +const CARRIED = (candidate) => [ + 'RELEASE-MANIFEST.json', + 'scripts/universe/protocol-contract.mjs', + 'docs/protocols/PROTOCOL-COVERAGE.json', + 'docs/acceptance/qualified-release-evidence.json', + candidate.journeyPath, + candidate.configurationPath, +]; + +function packed(name, entries) { + return archive(join(mkdtempSync(join(workdir, 'out-')), name), entries); +} + +test('a Signet-qualified Mainnet candidate qualifies from its extracted archive alone', async () => { + const candidate = signetQualifiedCandidate(); + const stage = stagedRelease(candidate); + // The checkout is gone before the archive is judged. + rmSync(candidate.root, { recursive: true, force: true }); + const file = packed('mempool-good.tar.gz', entriesOf(stage, CARRIED(candidate))); + assert.deepEqual(await qualifyArtifact(file, { commit: ARTIFACT_COMMIT, network: 'mainnet' }), []); +}); + +test('staging carries the complete evidence closure the envelope names, and nothing is left behind', () => { + const candidate = signetQualifiedCandidate(); + const stage = stagedRelease(candidate); + const closure = acceptanceEvidenceClosure(candidate.envelope).map((entry) => entry.path).sort(); + assert.deepEqual(closure, [candidate.configurationPath, candidate.journeyPath].sort()); + for (const relative of closure) { + assert.equal(readFileSync(join(stage, relative), 'utf8'), readFileSync(join(candidate.root, relative), 'utf8')); + } +}); + +test('an archive packed without docs, as the workflow used to, does not qualify', async () => { + const candidate = signetQualifiedCandidate(); + const stage = stagedRelease(candidate); + const file = packed('mempool-no-docs.tar.gz', entriesOf(stage, ['RELEASE-MANIFEST.json', 'scripts/universe/protocol-contract.mjs'])); + const problems = (await qualifyArtifact(file, { commit: ARTIFACT_COMMIT })).join('\n'); + assert.match(problems, /does not carry docs\/protocols\/PROTOCOL-COVERAGE\.json/); + assert.match(problems, /does not carry docs\/acceptance\/qualified-release-evidence\.json/); +}); + +test('an archive missing a nested evidence file does not qualify', async () => { + const candidate = signetQualifiedCandidate(); + const stage = stagedRelease(candidate); + const without = CARRIED(candidate).filter((relative) => relative !== candidate.journeyPath); + const problems = (await qualifyArtifact(packed('mempool-missing.tar.gz', entriesOf(stage, without)), { commit: ARTIFACT_COMMIT })).join('\n'); + assert.match(problems, /names unreadable evidence docs\/acceptance\/evidence\/signet\/run-1\/journeys\.json/); +}); + +test('tampered evidence bytes inside the archive do not qualify', async () => { + const candidate = signetQualifiedCandidate(); + const stage = stagedRelease(candidate); + const entries = entriesOf(stage, CARRIED(candidate)).map(([name, bytes]) => + name === candidate.journeyPath ? [name, Buffer.from(bytes.toString('utf8').replace('signet-fixture', 'signet-forged!'))] : [name, bytes]); + const problems = (await qualifyArtifact(packed('mempool-tampered.tar.gz', entries), { commit: ARTIFACT_COMMIT })).join('\n'); + assert.match(problems, /journeys\.json has SHA-256 [0-9a-f]{64}, not the recorded/); +}); + +test('a link in the evidence tree is refused before extraction', async () => { + const candidate = signetQualifiedCandidate(); + const stage = stagedRelease(candidate); + const entries = entriesOf(stage, CARRIED(candidate)).filter(([name]) => name !== candidate.journeyPath); + entries.push([candidate.journeyPath, { symlink: '../../../../../../etc/passwd' }]); + const problems = (await qualifyArtifact(packed('mempool-link.tar.gz', entries), { commit: ARTIFACT_COMMIT })).join('\n'); + assert.match(problems, /carries a link in its evidence tree/); +}); + +test('member names that are absolute or climb out of the release are refused', () => { + const required = ['RELEASE-MANIFEST.json', 'docs/protocols/PROTOCOL-COVERAGE.json', + 'docs/acceptance/qualified-release-evidence.json', 'scripts/universe/protocol-contract.mjs']; + assert.deepEqual(memberProblems(required, []), []); + assert.match(memberProblems([...required, '../outside.json'], []).join('\n'), /escapes the release directory/); + assert.match(memberProblems([...required, '/etc/cron.d/x'], []).join('\n'), /is absolute/); + assert.match(memberProblems([...required, 'docs/../../x'], []).join('\n'), /escapes/); +}); + +test('a stale revision does not qualify: the manifest commit and the envelope must name the release', async () => { + const candidate = signetQualifiedCandidate(); + const stale = stagedRelease(candidate, { commit: 'e'.repeat(40) }); + const staleFile = packed('mempool-stale.tar.gz', entriesOf(stale, CARRIED(candidate))); + const staleManifest = (await qualifyArtifact(staleFile, { commit: ARTIFACT_COMMIT })).join('\n'); + assert.match(staleManifest, /RELEASE-MANIFEST\.json names "e{40}", not the release commit d{40}/); + const good = stagedRelease(signetQualifiedCandidate()); + const other = (await qualifyArtifact(packed('mempool-other.tar.gz', entriesOf(good, CARRIED(candidate))), { commit: 'f'.repeat(40) })).join('\n'); + assert.match(other, /names artifact d{40}, not the intended artifact f{40}/); +}); + +test('staging refuses evidence outside docs/, traversal, absolute paths and wrong digests, and copies nothing', () => { + for (const [label, mutate, expected] of [ + ['outside docs', (entry) => { entry.path = 'scripts/universe/protocol-contract.mjs'; }, /not a plain path under docs\//], + ['traversal', (entry) => { entry.path = 'docs/../../outside.json'; }, /not a plain path under docs\/|escapes the evidence root/], + ['absolute', (entry) => { entry.path = '/etc/passwd'; }, /absolute evidence path|not a plain path/], + ['digest', (entry) => { entry.sha256 = '0'.repeat(64); }, /not the recorded/], + ['missing', (entry) => { entry.path = 'docs/acceptance/evidence/signet/run-1/absent.json'; }, /unreadable evidence/], + ]) { + const candidate = signetQualifiedCandidate(); + const envelope = JSON.parse(readFileSync(candidate.acceptancePath, 'utf8')); + mutate(envelope.rows[0].evidence[0]); + writeFileSync(candidate.acceptancePath, JSON.stringify(envelope)); + const stage = mkdtempSync(join(workdir, 'refused-')); + const report = stageAcceptance({ + manifestPath: candidate.manifestPath, acceptancePath: candidate.acceptancePath, acceptanceRoot: candidate.root, stageRoot: stage, + }); + assert.match(report.problems.join('\n'), expected, label); + assert.equal(existsSync(join(stage, 'docs')), false, `${label} staged files despite refusing`); + } +}); + +test('staging refuses an envelope that names no evidence at all', () => { + const candidate = signetQualifiedCandidate(); + writeFileSync(candidate.acceptancePath, JSON.stringify({ ...candidate.envelope, rows: [], candidate: { ...candidate.envelope.candidate, configurationProof: undefined } })); + const report = stageAcceptance({ + manifestPath: candidate.manifestPath, acceptancePath: candidate.acceptancePath, acceptanceRoot: candidate.root, stageRoot: mkdtempSync(join(workdir, 'empty-')), + }); + assert.match(report.problems.join('\n'), /names no evidence files/); +}); + +test('the workflow stages the closure, packs docs, and qualifies the packed archive before upload', () => { + const workflow = readFileSync(join(repositoryRoot, '.github', 'workflows', 'universe-release-artifact.yml'), 'utf8').replaceAll('\r\n', '\n'); + assert.match(workflow, /protocol-contract\.mjs --stage-acceptance "\$stage"/); + assert.match(workflow, /tar -czf "\$out" -C "\$stage" backend frontend scripts production docs RELEASE-MANIFEST\.json/); + const qualify = workflow.indexOf('- name: Qualify the packed archive on its own'); + const pack = workflow.indexOf('- name: Pack'); + const upload = workflow.indexOf('- name: Upload'); + assert.ok(pack > 0 && qualify > pack && upload > qualify, 'qualification must sit between packing and upload'); + assert.match(workflow.slice(qualify, upload), /qualify-artifact\.mjs[\s\S]*--commit '\$\{\{ steps\.sha\.outputs\.sha \}\}' --network mainnet/); +}); diff --git a/scripts/universe/release-gates.test.mjs b/scripts/universe/release-gates.test.mjs index e5cb2f96c5..ab493e3193 100644 --- a/scripts/universe/release-gates.test.mjs +++ b/scripts/universe/release-gates.test.mjs @@ -63,7 +63,9 @@ test('the artifact workflow requires candidate-bound acceptance before packing', assert.match(artifactWorkflow, /Qualify the release acceptance envelope/); assert.match(artifactWorkflow, /--expect-artifact-commit.*git rev-parse HEAD/); assert.match(artifactWorkflow, /--acceptance docs\/acceptance\/qualified-release-evidence\.json/); - assert.match(artifactWorkflow, /cp -a docs\/acceptance\/qualified-release-evidence\.json/); + // The envelope and its evidence closure are staged by the contract script; + // release-artifact.test.mjs proves the packed archive qualifies on its own. + assert.match(artifactWorkflow, /--stage-acceptance "\$stage"/); }); // Install the real function into a disposable release tree. Package download, From ad161bb7010a57feb3e1bd24a6ba553e8b123c51 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 20:57:03 +0000 Subject: [PATCH 08/11] chore(acceptance): regenerate the source-only operation matrix for the changed sources The prepared branch's annotation already left the matrix stale. Still FUNCTIONAL NO-GO with operationDenominatorReconciled false and zero real-network passes; nothing here is an acceptance result. Co-Authored-By: Claude Opus 5.5 --- .../operation-matrix-2026-09-06.json | 138 +++++++++--------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/docs/acceptance/operation-matrix-2026-09-06.json b/docs/acceptance/operation-matrix-2026-09-06.json index 85a39b6c7a..f0c52fc356 100644 --- a/docs/acceptance/operation-matrix-2026-09-06.json +++ b/docs/acceptance/operation-matrix-2026-09-06.json @@ -1926,9 +1926,9 @@ }, { "path": "backend/src/api/capabilities.routes.ts", - "sha256": "37b0250198ce047486fa47fc7398453dedced8bdacae44a886bfcc20d50ba31b", + "sha256": "ce069d5d5a7dabe3322bc3b7e1c762751740ea8ddf7718e458f4dbda1a882054", "sha256Encoding": "utf8-lf", - "bytes": 1008 + "bytes": 1397 }, { "path": "backend/src/api/data-studio/data-studio.routes.ts", @@ -2310,9 +2310,9 @@ }, { "path": "backend/src/config.ts", - "sha256": "54553601f59dab206af896774412a565009ff54595cb0af5b12b4f1745a99104", + "sha256": "ba418ef58ebc076b907abbaa1586249955180bde20cee2a251af30ddc968b5de", "sha256Encoding": "utf8-lf", - "bytes": 10886 + "bytes": 11014 }, { "path": "backend/src/index.ts", @@ -2538,9 +2538,9 @@ }, { "path": "frontend/src/app/components/master-page/master-page.component.ts", - "sha256": "e3f98227f2043ee7fedbebd9c40e0adb6d96cce1e147faab4e6f2905bc2d4d4f", + "sha256": "a24a6d9c0e804e41fc6d722226c78a929d895517c205748f52cbbe2ddd7e1888", "sha256Encoding": "utf8-lf", - "bytes": 11617 + "bytes": 11966 }, { "path": "frontend/src/app/components/mempool-block-view/mempool-block-view.component.ts", @@ -2934,21 +2934,21 @@ }, { "path": "frontend/src/app/universe/chain-dashboard/chain-dashboard.component.ts", - "sha256": "60af0361e93bacb8049a45f413b79be7ce846f957ae7a2d011609275a6cc3f32", + "sha256": "39ed49f689015a0dffaac516e60b80e0dc484c992869b3beab8c9ca7fefbb996", "sha256Encoding": "utf8-lf", - "bytes": 14099 + "bytes": 14330 }, { "path": "frontend/src/app/universe/chain-dashboard/chain-mining.component.ts", - "sha256": "a2200f9ba666651ce487f3087ca9a613090012c4e95b6e372bc1f70e9e1be085", + "sha256": "25646cfdd2c5258b680ccac8edc3cc057aceb3dbc1b6bccdd3de07b5ef36aceb", "sha256Encoding": "utf8-lf", - "bytes": 8161 + "bytes": 8392 }, { "path": "frontend/src/app/universe/chain-docs/chain-docs.component.ts", - "sha256": "b580c67422d4e0f32d149d7bec918f3e410255caa1ca04bc63cbacba549d113c", + "sha256": "c54cada20fd8b017697067e37b7f04d20b3ec75add21c171863d9a65fe8d2aca", "sha256Encoding": "utf8-lf", - "bytes": 3396 + "bytes": 3623 }, { "path": "frontend/src/app/universe/chain-graphs/chain-graphs.component.ts", @@ -3552,9 +3552,9 @@ }, { "path": "frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts", - "sha256": "2b30f8e7f8fc5a3351455b52da4c6a0587a800564fcb73aabf10c212407a0116", + "sha256": "c9c42dc5ad8f7b94a58585d5861a5d3adf9ee1c29dc2b1d3e6a6944895d92b27", "sha256Encoding": "utf8-lf", - "bytes": 31087 + "bytes": 31644 }, { "path": "frontend/src/app/universe/multichain-explorer/multichain-explorer.module.ts", @@ -4272,9 +4272,9 @@ }, { "path": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf", - "bytes": 42316 + "bytes": 42862 }, { "path": "frontend/src/app/universe/utxo-set/utxo-set.component.ts", @@ -4314,9 +4314,9 @@ }, { "path": "frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts", - "sha256": "d7c93c6094088c148074b901aca32f28149f39fbe35abf5c85303e6ae2d8ce0c", + "sha256": "9c9d7a410c56c992737de9afdc648bcba25cab97e2efcdb2a61bf09f8b0527c3", "sha256Encoding": "utf8-lf", - "bytes": 4531 + "bytes": 4791 }, { "path": "scripts/universe/acceptance-matrix.mjs", @@ -5065,7 +5065,7 @@ "status": "NOT TESTED", "currentSource": { "artifact": "frontend/src/app/components/master-page/master-page.component.ts", - "sha256": "e3f98227f2043ee7fedbebd9c40e0adb6d96cce1e147faab4e6f2905bc2d4d4f", + "sha256": "a24a6d9c0e804e41fc6d722226c78a929d895517c205748f52cbbe2ddd7e1888", "sha256Encoding": "utf8-lf" } }, @@ -6692,7 +6692,7 @@ "status": "NOT TESTED", "currentSource": { "artifact": "frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts", - "sha256": "d7c93c6094088c148074b901aca32f28149f39fbe35abf5c85303e6ae2d8ce0c", + "sha256": "9c9d7a410c56c992737de9afdc648bcba25cab97e2efcdb2a61bf09f8b0527c3", "sha256Encoding": "utf8-lf" } }, @@ -22935,7 +22935,7 @@ "status": "NOT TESTED", "currentSource": { "artifact": "frontend/src/app/universe/chain-dashboard/chain-dashboard.component.ts", - "sha256": "60af0361e93bacb8049a45f413b79be7ce846f957ae7a2d011609275a6cc3f32", + "sha256": "39ed49f689015a0dffaac516e60b80e0dc484c992869b3beab8c9ca7fefbb996", "sha256Encoding": "utf8-lf" } }, @@ -22987,7 +22987,7 @@ "status": "NOT TESTED", "currentSource": { "artifact": "frontend/src/app/universe/chain-dashboard/chain-mining.component.ts", - "sha256": "a2200f9ba666651ce487f3087ca9a613090012c4e95b6e372bc1f70e9e1be085", + "sha256": "25646cfdd2c5258b680ccac8edc3cc057aceb3dbc1b6bccdd3de07b5ef36aceb", "sha256Encoding": "utf8-lf" } }, @@ -23239,7 +23239,7 @@ "status": "NOT TESTED", "currentSource": { "artifact": "frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts", - "sha256": "2b30f8e7f8fc5a3351455b52da4c6a0587a800564fcb73aabf10c212407a0116", + "sha256": "c9c42dc5ad8f7b94a58585d5861a5d3adf9ee1c29dc2b1d3e6a6944895d92b27", "sha256Encoding": "utf8-lf" } }, @@ -25187,7 +25187,7 @@ "status": "NOT TESTED", "currentSource": { "artifact": "frontend/src/app/universe/chain-docs/chain-docs.component.ts", - "sha256": "b580c67422d4e0f32d149d7bec918f3e410255caa1ca04bc63cbacba549d113c", + "sha256": "c54cada20fd8b017697067e37b7f04d20b3ec75add21c171863d9a65fe8d2aca", "sha256Encoding": "utf8-lf" } } @@ -49321,7 +49321,7 @@ }, { "artifact": "frontend/src/app/components/master-page/master-page.component.ts", - "sha256": "e3f98227f2043ee7fedbebd9c40e0adb6d96cce1e147faab4e6f2905bc2d4d4f", + "sha256": "a24a6d9c0e804e41fc6d722226c78a929d895517c205748f52cbbe2ddd7e1888", "sha256Encoding": "utf8-lf" } ], @@ -49442,7 +49442,7 @@ }, { "artifact": "frontend/src/app/components/master-page/master-page.component.ts", - "sha256": "e3f98227f2043ee7fedbebd9c40e0adb6d96cce1e147faab4e6f2905bc2d4d4f", + "sha256": "a24a6d9c0e804e41fc6d722226c78a929d895517c205748f52cbbe2ddd7e1888", "sha256Encoding": "utf8-lf" } ], @@ -49563,7 +49563,7 @@ }, { "artifact": "frontend/src/app/components/master-page/master-page.component.ts", - "sha256": "e3f98227f2043ee7fedbebd9c40e0adb6d96cce1e147faab4e6f2905bc2d4d4f", + "sha256": "a24a6d9c0e804e41fc6d722226c78a929d895517c205748f52cbbe2ddd7e1888", "sha256Encoding": "utf8-lf" } ], @@ -49684,7 +49684,7 @@ }, { "artifact": "frontend/src/app/components/master-page/master-page.component.ts", - "sha256": "e3f98227f2043ee7fedbebd9c40e0adb6d96cce1e147faab4e6f2905bc2d4d4f", + "sha256": "a24a6d9c0e804e41fc6d722226c78a929d895517c205748f52cbbe2ddd7e1888", "sha256Encoding": "utf8-lf" } ], @@ -52062,7 +52062,7 @@ }, { "artifact": "frontend/src/app/universe/zcash-privacy/zcash-viewing-key-workspace.component.ts", - "sha256": "d7c93c6094088c148074b901aca32f28149f39fbe35abf5c85303e6ae2d8ce0c", + "sha256": "9c9d7a410c56c992737de9afdc648bcba25cab97e2efcdb2a61bf09f8b0527c3", "sha256Encoding": "utf8-lf" } ], @@ -80155,7 +80155,7 @@ }, { "artifact": "frontend/src/app/universe/chain-dashboard/chain-mining.component.ts", - "sha256": "a2200f9ba666651ce487f3087ca9a613090012c4e95b6e372bc1f70e9e1be085", + "sha256": "25646cfdd2c5258b680ccac8edc3cc057aceb3dbc1b6bccdd3de07b5ef36aceb", "sha256Encoding": "utf8-lf" } ], @@ -80393,7 +80393,7 @@ }, { "artifact": "frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts", - "sha256": "2b30f8e7f8fc5a3351455b52da4c6a0587a800564fcb73aabf10c212407a0116", + "sha256": "c9c42dc5ad8f7b94a58585d5861a5d3adf9ee1c29dc2b1d3e6a6944895d92b27", "sha256Encoding": "utf8-lf" } ], @@ -89825,7 +89825,7 @@ }, { "artifact": "frontend/src/app/universe/chain-docs/chain-docs.component.ts", - "sha256": "b580c67422d4e0f32d149d7bec918f3e410255caa1ca04bc63cbacba549d113c", + "sha256": "c54cada20fd8b017697067e37b7f04d20b3ec75add21c171863d9a65fe8d2aca", "sha256Encoding": "utf8-lf" } ], @@ -89917,7 +89917,7 @@ }, { "artifact": "frontend/src/app/universe/chain-docs/chain-docs.component.ts", - "sha256": "b580c67422d4e0f32d149d7bec918f3e410255caa1ca04bc63cbacba549d113c", + "sha256": "c54cada20fd8b017697067e37b7f04d20b3ec75add21c171863d9a65fe8d2aca", "sha256Encoding": "utf8-lf" } ], @@ -101972,7 +101972,7 @@ }, { "artifact": "backend/src/api/capabilities.routes.ts", - "sha256": "37b0250198ce047486fa47fc7398453dedced8bdacae44a886bfcc20d50ba31b", + "sha256": "ce069d5d5a7dabe3322bc3b7e1c762751740ea8ddf7718e458f4dbda1a882054", "sha256Encoding": "utf8-lf" } ], @@ -127842,7 +127842,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -127922,7 +127922,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -127981,7 +127981,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -128039,7 +128039,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -128099,7 +128099,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -128158,7 +128158,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -128217,7 +128217,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -128276,7 +128276,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -128344,7 +128344,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -128447,7 +128447,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -128536,7 +128536,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -128627,7 +128627,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -128721,7 +128721,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -128814,7 +128814,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -128909,7 +128909,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -129002,7 +129002,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -129093,7 +129093,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -129183,7 +129183,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -129278,7 +129278,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -129371,7 +129371,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -129466,7 +129466,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -130189,7 +130189,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130245,7 +130245,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130301,7 +130301,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130357,7 +130357,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130413,7 +130413,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130469,7 +130469,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130525,7 +130525,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130581,7 +130581,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130637,7 +130637,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130702,7 +130702,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -130787,7 +130787,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { @@ -130863,7 +130863,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130919,7 +130919,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" } ], @@ -130984,7 +130984,7 @@ }, { "artifact": "frontend/src/app/universe/universe-api.service.ts", - "sha256": "40002f731b07bf8fe0c3b3f8ae534e186ee26a7bf169312219b81f4067fecee9", + "sha256": "a258ccd4094f8fb6dc475f666faba2c4c508cc1940a43da3c45790bc74bff55a", "sha256Encoding": "utf8-lf" }, { From 1d32acbdb52bcba588f913c9c843fc210f91fbcb Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 21:00:46 +0000 Subject: [PATCH 09/11] docs(audit): record the 2026-09-23 implementation disposition and the verified Fulcrum outage cause Co-Authored-By: Claude Opus 5.5 --- .../mainnet-20260923/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/implementation-prep/mainnet-20260923/README.md b/docs/implementation-prep/mainnet-20260923/README.md index 814f72a904..88ed25f6b3 100644 --- a/docs/implementation-prep/mainnet-20260923/README.md +++ b/docs/implementation-prep/mainnet-20260923/README.md @@ -10,6 +10,23 @@ The SERVER checkout is `D:\universe\mempool\mempool`, whose HEAD file names main The created SERVER handoff directory is `D:\universe\mempool\audits\implementation-prep-20260923-1745\mempool_HANDOFF_2026-09-23`. Directory creation alone does not prove a prompt or ZIP was saved; consult the final handoff delivery receipt. Workspace AGENTS.md was read but is not redistributed because it contains credentials. Treat secrets as credentials, never as report content. +## Implementation status, 2026-09-23 evening + +Implemented on `implement/mainnet-20260923` (PR #136) from the prepared revision `161b7bdd0`. Unit, integration-style and archive tests pass as listed in the PR. **None of this is functional acceptance, and nothing was released.** + +| ID | Disposition | Functional status | +|---|---|---| +| M23-NET | IMPLEMENTED. Typed unavailable result, owning-contract keys and networks, every caller updated; the source marker is replaced by rationale | Unit and consumer tests PASS; supported-network journey NOT TESTED | +| M23-HEALTH | IMPLEMENTED in `capabilities.mining.ts` (lag bound `MEMPOOL.MINING_MAX_BEHIND_TIP`, fresh same-network Core reading, `unknown` state). The live outage cause is verified and repaired (below); an outstanding note replaces the marker | Unit and report tests PASS; Signet API-to-UI NOT TESTED | +| M23-PACK | IMPLEMENTED. `--stage-acceptance`, `docs` in the archive, `qualify-artifact.mjs` before upload; the marker is replaced by rationale | Real-archive tests PASS; a real workflow run needs a qualified envelope, which does not exist | +| M23-BASE | DONE. Worktree `D:\universe\mempool\.worktrees\mainnet-execution-20260923`, Node 24.19.0, npm 11.17.0; running backend 537235052, overlay fcdc2e2f | n/a | +| M23-AUTHORITY | NOT STARTED in owning repositories | BLOCKED: the production overlay reports every Bitcoin protocol unavailable on Signet and Testnet and every Dogecoin and Zcash protocol unavailable on Testnet | +| M23-COVERAGE | Matrix regenerated for changed sources; still `operationDenominatorReconciled: false` | NOT TESTED | +| M23-ACCEPT | NOT EXECUTED | BLOCKED on M23-AUTHORITY | +| M23-RELEASE | NOT EXECUTED; `gate_qualified_acceptance` correctly refuses without a qualified envelope | BLOCKED | + +**F-M23-04 cause, verified and repaired.** Fulcrum (`universe-fulcrum`, 127.0.0.1:50001), the explorer's electrum address index, stopped cleanly at 2026-09-22T13:05Z. The Bitcoin Core migration to the OVH node stopped `bitcoin.service`, and Fulcrum and its dependents went down in the same cascade. Nothing restarted them. `bitcoin.service` is now the RPC bridge to the migrated node, so starting Fulcrum was safe. It was restarted at 2026-09-23T20:55Z, caught up 175 blocks, and `addressLookup` reports ready at 968318. The explorer checkpoint stayed at 968172, the migration's frozen height, because the block loop was waiting on the dead index. + ## Source annotation index | ID | Actual source anchor | Dependencies | Preparation | Functional status | From b8fe3b8691fa4bf62fc0ee7d45c37f211238b35b Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 21:46:55 +0000 Subject: [PATCH 10/11] fix(rbf): check a restored RBF cache with a bounded pool instead of one read at a time With an electrum or Core backend, restoring the RBF cache read every unexpired cached transaction sequentially before the HTTP server listened. On 2026-09-23 that was 8,044 reads through the new Core RPC tunnel, and the explorer API stayed down for about half an hour on each restart while the watchdog kept restarting a stalled block loop. Eight concurrent reads, each still caught on its own, well inside the shared RPC budget. Co-Authored-By: Claude Opus 5.5 --- .../rbf-cache-restore-concurrency.test.ts | 51 +++++++++++++++++++ backend/src/api/rbf-cache.ts | 28 +++++++--- 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 backend/src/__tests__/rbf-cache-restore-concurrency.test.ts diff --git a/backend/src/__tests__/rbf-cache-restore-concurrency.test.ts b/backend/src/__tests__/rbf-cache-restore-concurrency.test.ts new file mode 100644 index 0000000000..0ff8d8c896 --- /dev/null +++ b/backend/src/__tests__/rbf-cache-restore-concurrency.test.ts @@ -0,0 +1,51 @@ +/** + * Restoring the RBF cache checks every unexpired cached transaction against + * the node before the HTTP server listens. Read one at a time, 8,044 of them + * held the explorer API down for about half an hour per restart on + * 2026-09-23. These pin the replacement: every transaction is still read, + * never more than RBF_CHECK_CONCURRENCY at once, and a failed read does not + * stop the rest. + */ +const inFlight = { now: 0, max: 0, calls: 0 }; + +jest.mock('../api/bitcoin/bitcoin-api-factory', () => ({ + __esModule: true, + default: { + $getRawTransaction: async (txid: string) => { + inFlight.calls += 1; + inFlight.now += 1; + inFlight.max = Math.max(inFlight.max, inFlight.now); + await new Promise((resolve) => setTimeout(resolve, 2)); + inFlight.now -= 1; + if (txid.endsWith('f')) {throw new Error('404');} + return { txid, status: { confirmed: false } }; + }, + }, +})); + +import config from '../config'; + +// testSetup replaces rbf-cache with an empty module; this suite needs the real one. +// Its constructor starts a ten minute cleanup interval, which would keep Jest +// from exiting, so only setInterval is faked; the reads still use real timeouts. +jest.useFakeTimers({ doNotFake: ['setTimeout', 'clearTimeout', 'setImmediate', 'nextTick', 'queueMicrotask', 'Date'] }); +const { default: rbfCache, RBF_CHECK_CONCURRENCY } = jest.requireActual('../api/rbf-cache'); + +describe('RBF cache restore against an electrum or Core backend', () => { + const backend = config.MEMPOOL.BACKEND; + afterAll(() => { config.MEMPOOL.BACKEND = backend; jest.clearAllTimers(); jest.useRealTimers(); }); + + it('reads every cached transaction with at most the bounded number in flight', async () => { + config.MEMPOOL.BACKEND = 'electrum'; + const count = 120; + const txs = Array.from({ length: count }, (_, i) => { + const txid = i.toString(16).padStart(63, '0') + (i % 10 === 0 ? 'f' : '0'); + return { value: { txid, vin: [], vout: [], fee: 0, weight: 400, vsize: 100, adjustedVsize: 100, sigops: 0, feePerVsize: 0, effectiveFeePerVsize: 0 } }; + }); + await rbfCache.load({ txs, trees: [], expiring: [], mempool: {}, spendMap: new Map() }); + expect(inFlight.calls).toBe(count); + expect(inFlight.max).toBeGreaterThan(1); + expect(inFlight.max).toBeLessThanOrEqual(RBF_CHECK_CONCURRENCY); + expect(inFlight.now).toBe(0); + }); +}); diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index 2197b0d4cb..54c01f79a9 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -6,6 +6,9 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface'; import { Common } from './common'; import redisCache from './redis-cache'; +/** Concurrent RPC reads when a restored RBF cache is checked against the node. */ +export const RBF_CHECK_CONCURRENCY = 8; + export interface RbfTransaction extends TransactionStripped { rbf?: boolean; mined?: boolean; @@ -580,14 +583,27 @@ class RbfCache { } } } else { + // Read with a small fixed pool rather than one at a time. This runs + // before the HTTP server listens, and on 2026-09-23 a cache of 8,044 + // unexpired transactions read sequentially through the Core RPC tunnel + // held the whole API down for about half an hour on every restart. The + // pool stays well inside the shared RPC budget. const txs: IEsploraApi.Transaction[] = []; - for (const txid of txids) { - try { - const tx = await bitcoinApi.$getRawTransaction(txid, true, false); - txs.push(tx); - } catch (err) { - // some 404s are expected, so continue quietly + let next = 0; + const worker = async (): Promise => { + while (next < txids.length) { + const txid = txids[next++]; + try { + txs.push(await bitcoinApi.$getRawTransaction(txid, true, false)); + } catch (err) { + // some 404s are expected, so continue quietly + } } + }; + try { + await Promise.all(Array.from({ length: Math.min(RBF_CHECK_CONCURRENCY, txids.length) }, () => worker())); + } catch (err) { + logger.err('failed to check cached rbf transactions: ' + (err instanceof Error ? err.message : err)); } processTxs(txs); } From 62ce4e430f843d876a42abd89de001bf6ea0fbc3 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Wed, 23 Sep 2026 21:55:52 +0000 Subject: [PATCH 11/11] fix(blocks): read a block the mempool no longer holds in one Core request Against Core (electrum or none backends), every block transaction missing from the mempool cost two RPC round trips: the transaction and its block header. Since Core moved behind a 115 ms tunnel on 2026-09-22 that is about 0.5 s per transaction and roughly thirty minutes for a block the mempool no longer holds, which is the main loop watchdog's whole budget. The explorer stalled at 968172 and the watchdog restarted it repeatedly. When more than 50 transactions are missing, $getTransactionsExtended now builds them from one verbose getblock (status from the block, fee from Core's own field, no prevouts, exactly as the per-transaction path did with prevouts off), and falls back to per-transaction reads if that read fails. Stale blocks and esplora keep their existing paths. Co-Authored-By: Claude Opus 5.5 --- .../bitcoin-transaction-status.test.ts | 27 +++++++++++++++++ .../bitcoin/bitcoin-api-abstract-factory.ts | 2 ++ backend/src/api/bitcoin/bitcoin-api.ts | 29 +++++++++++++++++++ backend/src/api/blocks.ts | 24 +++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/backend/src/__tests__/bitcoin-transaction-status.test.ts b/backend/src/__tests__/bitcoin-transaction-status.test.ts index 9fec3dac2c..cec9c2a72a 100644 --- a/backend/src/__tests__/bitcoin-transaction-status.test.ts +++ b/backend/src/__tests__/bitcoin-transaction-status.test.ts @@ -97,4 +97,31 @@ describe('Bitcoin Core transaction confirmation identity', () => { expect(transaction.status).toEqual({ confirmed: true, block_height: 0, block_hash: HASH, block_time: BLOCK.time }); expect(client.getBlockHeader).not.toHaveBeenCalled(); }); + it('reads a whole block for ingestion in one request, with block status and Core fees, and no per-transaction reads', async () => { + const { api, client } = fixture(); + const spend = { ...RAW, txid: 'a'.repeat(64), vin: [{ txid: TXID, vout: 0, scriptSig: { hex: '' }, sequence: 1 }], fee: 0.00000321 }; + client.getBlock.mockResolvedValue({ ...BLOCK, tx: [RAW, spend] }); + const transactions = await api.$getTxsForBlockWithoutPrevouts(HASH); + expect(transactions.map(tx => tx.txid)).toEqual([TXID, 'a'.repeat(64)]); + for (const transaction of transactions) { + expect(transaction.status).toEqual({ confirmed: true, block_height: BLOCK.height, block_hash: HASH, block_time: BLOCK.time }); + } + expect(transactions[1].fee).toBe(321); + expect(transactions[1].vin[0].prevout).toBeNull(); + expect(client.getBlock).toHaveBeenCalledTimes(1); + expect(client.getBlock).toHaveBeenCalledWith(HASH, 2); + expect(client.getRawTransaction).not.toHaveBeenCalled(); + expect(client.getBlockHeader).not.toHaveBeenCalled(); + expect(client.getMempoolEntry).not.toHaveBeenCalled(); + }); + + it.each([ + { ...BLOCK, confirmations: -1, tx: [RAW] }, + { ...BLOCK, hash: 'f'.repeat(64), tx: [RAW] }, + { ...BLOCK, tx: undefined }, + ])('refuses a stale, mismatched or transactionless block for the one-request read: %j', async block => { + const { api, client } = fixture(); + client.getBlock.mockResolvedValue(block); + await expect(api.$getTxsForBlockWithoutPrevouts(HASH)).rejects.toThrow('not an active-chain block'); + }); }); diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index eaa079eb6d..65239d18f1 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -13,6 +13,8 @@ export interface AbstractBitcoinApi { $getBlockHashTip(): Promise; $getTxIdsForBlock(hash: string, fallbackToCore?: boolean): Promise; $getTxsForBlock(hash: string, fallbackToCore?: boolean): Promise; + /** Core-backed APIs only: a block's transactions from one verbose read, without prevouts. */ + $getTxsForBlockWithoutPrevouts?(hash: string): Promise; $getBlockHash(height: number): Promise; $getBlockHeader(hash: string): Promise; $getBlock(hash: string): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 21eb063df0..4fa8ffbf59 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -128,6 +128,35 @@ class BitcoinApi implements AbstractBitcoinApi { return transactions; } + /** + * Every transaction of a block from one verbose block read, without + * prevouts: the same documents the per-transaction path builds with + * addPrevout off, with the status taken from the block and the fee from + * Core's own per-transaction field. The per-transaction path costs two RPC + * round trips per transaction (the transaction and its block header); with + * Core behind a 115 ms tunnel that was about thirty minutes for a block the + * mempool no longer held, the whole of the main loop watchdog's budget. + * + * @asyncUnsafe + */ + async $getTxsForBlockWithoutPrevouts(hash: string): Promise { + const verboseBlock: IBitcoinApi.VerboseBlock = await this.bitcoindClient.getBlock(hash, 2); + if (!verboseBlock || verboseBlock.hash !== hash || !Array.isArray(verboseBlock.tx) || verboseBlock.confirmations === -1) { + throw new Error('Block ' + hash + ' is not an active-chain block with transactions'); + } + const status = { confirmed: true, block_height: verboseBlock.height, block_hash: hash, block_time: verboseBlock.time }; + const transactions: IEsploraApi.Transaction[] = []; + for (const tx of verboseBlock.tx) { + const converted = await this.$convertTransaction(tx, false, false, false, status); + const fee = (tx as { fee?: unknown }).fee; + if (typeof fee === 'number' && Number.isFinite(fee)) { + converted.fee = Math.round(fee * 100_000_000); + } + transactions.push(converted); + } + return transactions; + } + $getRawBlock(hash: string): Promise { return this.bitcoindClient.getBlock(hash, 0) .then((raw: string) => Buffer.from(raw, 'hex')); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 9e95c24f4c..61613d77e7 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -37,6 +37,12 @@ import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; import database from '../database'; import { getBlockFirstSeenFromLogs, getOldestLogTimestampFromLogs, scanLogsForBlocksFirstSeen } from '../utils/file-read'; +/** + * More transactions than this missing from the mempool, and a Core-backed + * block read fetches the whole block once instead of one transaction at a time. + */ +export const CORE_BULK_BLOCK_READ_THRESHOLD = 50; + class Blocks { private blocks: BlockExtended[] = []; private blockSummaries: BlockSummary[] = []; @@ -164,6 +170,24 @@ class Blocks { } } + // Against Core (electrum or none), a block the mempool no longer holds is + // read in one verbose request rather than two round trips per transaction. + const missingFromMempool = txIds.length - totalFound; + if (!isEsplora && !stale && !onlyCoinbase && missingFromMempool > CORE_BULK_BLOCK_READ_THRESHOLD + && bitcoinApi.$getTxsForBlockWithoutPrevouts) { + try { + const rawTransactions = await bitcoinApi.$getTxsForBlockWithoutPrevouts(blockHash); + for (const tx of rawTransactions) { + if (!transactionMap[tx.txid]) { + transactionMap[tx.txid] = addMempoolData ? transactionUtils.extendMempoolTransaction(tx) : transactionUtils.extendTransaction(tx); + totalFound++; + } + } + } catch (e) { + logger.err(`Cannot read block ${blockHash} in one request, falling back to one read per transaction. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + // Fetch remaining txs individually for (const txid of txIds.filter(txid => !transactionMap[txid])) { if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam