diff --git a/docs/packages/sdk/bsv-sdk.md b/docs/packages/sdk/bsv-sdk.md index 306afee2a..bf025434e 100644 --- a/docs/packages/sdk/bsv-sdk.md +++ b/docs/packages/sdk/bsv-sdk.md @@ -3,7 +3,7 @@ id: bsv-sdk title: '@bsv/sdk' kind: package domain: sdk -version: '2.2.18' +version: '2.3.0' npm: '@bsv/sdk' last_updated: '2026-07-31' last_verified: '2026-07-31' diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index 09de505ab..ee746bc6a 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-client title: '@bsv/wallet-toolbox-client' kind: package domain: wallet -version: '2.4.22' +version: '2.5.0' last_updated: '2026-07-31' last_verified: '2026-07-31' review_cadence_days: 30 diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index 9861dcd2f..2be3ba923 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-mobile title: '@bsv/wallet-toolbox-mobile' kind: package domain: wallet -version: '2.4.22' +version: '2.5.0' last_updated: '2026-07-31' last_verified: '2026-07-31' review_cadence_days: 30 diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index 3b8824cb5..8243ae77e 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -4,7 +4,7 @@ title: '@bsv/wallet-toolbox' kind: package domain: wallet npm: '@bsv/wallet-toolbox' -version: '2.4.22' +version: '2.5.0' last_updated: '2026-07-31' last_verified: '2026-07-31' review_cadence_days: 30 diff --git a/docs/reference/dependency-policy.md b/docs/reference/dependency-policy.md index 9c3d7aa1f..8c6453b7d 100644 --- a/docs/reference/dependency-policy.md +++ b/docs/reference/dependency-policy.md @@ -3,8 +3,8 @@ id: dependency-release-policy title: 'Dependency and Release Policy' kind: reference version: '1.3.0' -last_updated: '2026-07-30' -last_verified: '2026-07-30' +last_updated: '2026-08-04' +last_verified: '2026-08-04' review_cadence_days: 30 status: stable tags: [reference, dependencies, security, releases] @@ -121,10 +121,10 @@ unresolvable public declarations. The root workspace carries two narrow audited dependency overrides: -- Jest 30.4.2 still constrains its reporting and coverage graph to minimatch - releases that require `brace-expansion` 1.x/2.x, while - GHSA-mh99-v99m-4gvg is fixed only in `brace-expansion` 5.0.8. The workspace - substitutes 5.0.8 until Jest adopts minimatch 10.2.5 or newer. +- Jest 30.4.2 still constrains parts of its reporting and coverage graph to + minimatch releases with older `brace-expansion` ranges. The follow-up + GHSA-rgw5-rvv9-x895 requires `brace-expansion` 5.0.9, so the workspace + substitutes 5.0.9 until every supported path resolves it natively. - Stryker 9.6.1's current `typed-rest-client@2.3.1` dependency pins vulnerable `qs@6.15.1` exactly. A parent-scoped substitution selects 6.15.3 until upstream accepts 6.15.2 or newer. This replaces a fragile lock-only @@ -139,13 +139,14 @@ frozen graph stayed clean without it. The machine-readable registry now maps every remaining selector and exact value to its exception. CI rejects a new, changed, stale, expired, unowned, or upstream-unlinked override. -Wave 38 repeated the removal rehearsal by regenerating every standalone lock +Wave 39 repeated the removal rehearsal by regenerating every standalone lock without its `gaxios`, `uuid`, and `brace-expansion` substitutions and checking the natural dependency graph. It also rechecked the root Jest/minimatch, -typed-rest-client/qs, and isolated Redocly closures. All 19 remaining -selectors still prevent a reproduced vulnerable resolution or preserve the -governed reproducible generator, so none can be removed safely yet. The -method, result, count, and next rehearsal are enforced in +typed-rest-client/qs, and isolated Redocly closures, then refreshed affected +locks for the current `brace-expansion`, `fast-uri`, `ip-address`, and +`socket.io-parser` advisories. All 19 remaining selectors still prevent a +reproduced vulnerable resolution or preserve the governed reproducible +generator, so none can be removed safely yet. The method, result, count, and next rehearsal are enforced in `governance/dependency-release-policy.json`. The independently locked OpenAPI generator also carries a narrow Redocly diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 165937212..28df7e268 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -23,39 +23,39 @@ and clean-consumer tests remain the executable type authority. ## Current release boundary -| Package | npm baseline | Source | Candidate | API | Migration | -| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@bsv/402-pay` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. | -| `@bsv/air-gap` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | -| `@bsv/amountinator` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | -| `@bsv/auth` | `0.1.1` | `0.1.3` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | -| `@bsv/auth-express-middleware` | `2.1.2` | `2.1.7` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No consumer migration is required; existing public CORS defaults and middleware APIs are retained, authentication callback failures now produce a controlled HTTP error, and Express 4 and 5 applications use their own peer-provided Express installation. | -| `@bsv/authsocket` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/messaging/authsocket.md) | Valid traffic and the wire contract are unchanged. A socket is now disconnected when its authentication processing exceeds the concurrency limit or a callback fails; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | -| `@bsv/authsocket-client` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/messaging/authsocket-client.md) | Valid traffic and supported imports are unchanged. The client now disconnects from a server that causes authentication failure or exceeds the concurrency limit; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | -| `@bsv/btms` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms.md) | No consumer migration is required; token and lookup wire contracts are unchanged. | -| `@bsv/btms-permission-module` | `1.1.1` | `1.1.3` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | -| `@bsv/did` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | -| `@bsv/did-client` | `1.2.1` | `1.2.3` | patch | [API and usage](../packages/helpers/did-client.md) | No consumer migration is required; DID client APIs, encodings, and supported imports are unchanged. | -| `@bsv/fund-wallet` | `1.4.1` | `1.4.3` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. | -| `@bsv/gasp` | `1.3.1` | `1.3.5` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. | -| `@bsv/message-box-client` | `2.2.2` | `2.2.6` | patch | [API and usage](../packages/messaging/message-box-client.md) | No consumer migration is required; Message Box protocol and client entry points are unchanged. | -| `@bsv/overlay` | `2.2.1` | `2.2.7` | patch | [API and usage](../packages/overlays/overlay.md) | No consumer migration is required; existing imports, submission results, notification contracts, storage order, and network behavior remain unchanged. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/overlays/overlay-discovery-services.md) | No consumer migration is required; discovery records and public network behavior are unchanged. | -| `@bsv/overlay-express` | `2.4.2` | `2.4.9` | patch | [API and usage](../packages/overlays/overlay-express.md) | No consumer migration is required; wildcard credential-free public access remains the default and runtimes may opt into the new close method. | -| `@bsv/overlay-topics` | `1.6.1` | `1.6.8` | patch | [API and usage](../packages/overlays/overlay-topics.md) | No consumer migration is required; topic IDs, lookup contracts, and persisted formats are unchanged. | -| `@bsv/paymail` | `2.4.2` | `2.4.6` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. | -| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. | -| `@bsv/sdk` | `2.2.0` | `2.2.18` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No consumer migration is required. BEEF ordering, validation results, serialized bytes, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | -| `@bsv/simple` | `0.4.1` | `0.4.8` | patch | [API and usage](../packages/helpers/simple.md) | No consumer migration is required; the browser and server entry points remain compatible. | -| `@bsv/templates` | `1.9.1` | `1.9.6` | patch | [API and usage](../packages/helpers/templates.md) | No consumer migration is required; template APIs, supported imports, and generated script semantics are unchanged. | -| `@bsv/teranode-listener` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. | -| `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | -| `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | -| `@bsv/wallet-relay` | `0.2.2` | `0.3.4` | minor | [API and usage](../packages/wallet/wallet-relay.md) | QRPairingCode now renders a native button and accepts button wrapper attributes. Existing className, style, data, and ARIA props continue to work; update div-specific wrapper selectors or explicitly typed div event handlers. Express integrations now use the host application's matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.4.4` | `2.4.22` | patch | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. SQLite/MySQL and IndexedDB indexes are added by the existing migration paths; wallet data and results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged. | -| `@bsv/wallet-toolbox-client` | `2.4.4` | `2.4.22` | patch | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required; IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. | -| `@bsv/wallet-toolbox-mobile` | `2.4.4` | `2.4.22` | patch | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required; BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. | -| `create-bsv-app` | `1.0.2` | `1.0.4` | patch | [API and usage](../packages/helpers/create-bsv-app.md) | No consumer migration is required; generated application structure and CLI behavior are unchanged. | +| Package | npm baseline | Source | Candidate | API | Migration | +| --------------------------------- | ------------ | ------- | --------- | --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `@bsv/402-pay` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. | +| `@bsv/air-gap` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | +| `@bsv/amountinator` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | +| `@bsv/auth` | `0.1.1` | `0.1.3` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | +| `@bsv/auth-express-middleware` | `2.1.2` | `2.1.7` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No consumer migration is required; existing public CORS defaults and middleware APIs are retained, authentication callback failures now produce a controlled HTTP error, and Express 4 and 5 applications use their own peer-provided Express installation. | +| `@bsv/authsocket` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/messaging/authsocket.md) | Valid traffic and the wire contract are unchanged. A socket is now disconnected when its authentication processing exceeds the concurrency limit or a callback fails; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | +| `@bsv/authsocket-client` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/messaging/authsocket-client.md) | Valid traffic and supported imports are unchanged. The client now disconnects from a server that causes authentication failure or exceeds the concurrency limit; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | +| `@bsv/btms` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms.md) | No consumer migration is required; token and lookup wire contracts are unchanged. | +| `@bsv/btms-permission-module` | `1.1.1` | `1.1.3` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | +| `@bsv/did` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | +| `@bsv/did-client` | `1.2.1` | `1.2.3` | patch | [API and usage](../packages/helpers/did-client.md) | No consumer migration is required; DID client APIs, encodings, and supported imports are unchanged. | +| `@bsv/fund-wallet` | `1.4.1` | `1.4.3` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. | +| `@bsv/gasp` | `1.3.1` | `1.3.5` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. | +| `@bsv/message-box-client` | `2.2.2` | `2.2.6` | patch | [API and usage](../packages/messaging/message-box-client.md) | No consumer migration is required; Message Box protocol and client entry points are unchanged. | +| `@bsv/overlay` | `2.2.1` | `2.2.7` | patch | [API and usage](../packages/overlays/overlay.md) | No consumer migration is required; existing imports, submission results, notification contracts, storage order, and network behavior remain unchanged. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/overlays/overlay-discovery-services.md) | No consumer migration is required; discovery records and public network behavior are unchanged. | +| `@bsv/overlay-express` | `2.4.2` | `2.4.9` | patch | [API and usage](../packages/overlays/overlay-express.md) | No consumer migration is required; wildcard credential-free public access remains the default and runtimes may opt into the new close method. | +| `@bsv/overlay-topics` | `1.6.1` | `1.6.8` | patch | [API and usage](../packages/overlays/overlay-topics.md) | No consumer migration is required; topic IDs, lookup contracts, and persisted formats are unchanged. | +| `@bsv/paymail` | `2.4.2` | `2.4.6` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. | +| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. | +| `@bsv/sdk` | `2.2.18` | `2.3.0` | minor | [API and usage](../packages/sdk/bsv-sdk.md) | No consumer migration is required. The new APIs are additive; BEEF ordering and bytes, proof validity, signatures, synchronous signing, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | +| `@bsv/simple` | `0.4.1` | `0.4.8` | patch | [API and usage](../packages/helpers/simple.md) | No consumer migration is required; the browser and server entry points remain compatible. | +| `@bsv/templates` | `1.9.1` | `1.9.6` | patch | [API and usage](../packages/helpers/templates.md) | No consumer migration is required; template APIs, supported imports, and generated script semantics are unchanged. | +| `@bsv/teranode-listener` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. | +| `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | +| `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | +| `@bsv/wallet-relay` | `0.2.2` | `0.3.4` | minor | [API and usage](../packages/wallet/wallet-relay.md) | QRPairingCode now renders a native button and accepts button wrapper attributes. Existing className, style, data, and ARIA props continue to work; update div-specific wrapper selectors or explicitly typed div event handlers. Express integrations now use the host application's matching Express runtime and type graph. | +| `@bsv/wallet-toolbox` | `2.4.22` | `2.5.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Storage-provider additions are backward-compatible with fallbacks, existing databases use the normal migration path, and wallet results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged. | +| `@bsv/wallet-toolbox-client` | `2.4.22` | `2.5.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required; older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. | +| `@bsv/wallet-toolbox-mobile` | `2.4.22` | `2.5.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required; older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. | +| `create-bsv-app` | `1.0.2` | `1.0.4` | patch | [API and usage](../packages/helpers/create-bsv-app.md) | No consumer migration is required; generated application structure and CLI behavior are unchanged. | `none` means the source manifest matches the recorded npm baseline. Any other value is an unpublished candidate. Publication, tags, releases, registry @@ -316,8 +316,8 @@ CLI entry points: `{"fund-metanet":"./dist/index.mjs"}`. - Package documentation: [docs/packages/sdk/bsv-sdk.md](../packages/sdk/bsv-sdk.md) - Source: [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) -- Release note: Accumulates security and correctness hardening, transaction and action-batch performance work, strict package contracts, safer text and telemetry handling, and caches unchanged BEEF dependency-sort results so repeated wallet known-txid preparation avoids a full topological sort. -- Migration: No consumer migration is required. BEEF ordering, validation results, serialized bytes, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. +- Release note: Adds batched proven-transaction BEEF assembly and deferred-then-compound Merkle proof validation, reuses shared proof hashes and BRC-42 counterparty secrets, and accelerates optional-backend P2PKH public-key emission. +- Migration: No consumer migration is required. The new APIs are additive; BEEF ordering and bytes, proof validity, signatures, synchronous signing, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | @@ -477,8 +477,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Adds fee-aware read-only legacy createAction planning, one-transaction funding claims, exact reservation filtering, Knex and IndexedDB funding indexes, faster known-txid and proof handling, and privacy-safe timings for every material funding and BEEF phase. -- Migration: No consumer migration is required. SQLite/MySQL and IndexedDB indexes are added by the existing migration paths; wallet data and results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged. +- Release note: Makes the successful fragmented createAction path atomic and set-based, overlaps batched proof reads with persistence, batch-validates compound proofs and canonical P2PKH signatures, shares BRC-42 derivation work, removes unused commit reads, bulk-inserts outputs, coalesces authenticated timestamp-only Knex session touches, and adds timings for every remaining material phase. +- Migration: No consumer migration is required. Storage-provider additions are backward-compatible with fallbacks, existing databases use the normal migration path, and wallet results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -491,8 +491,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Carries the lockstep browser build with read-only funding planning, atomic claims, IndexedDB funding indexes, BEEF-history caching, and expanded privacy-safe createAction timings. -- Migration: No consumer migration is required; IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. +- Release note: Carries the lockstep browser build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, and complete privacy-safe createAction timings. +- Migration: No consumer migration is required; older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -503,8 +503,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Carries the lockstep mobile build with read-only funding planning, atomic claims, faster BEEF-history handling, and expanded privacy-safe createAction timings. -- Migration: No consumer migration is required; BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. +- Release note: Carries the lockstep mobile build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, and complete privacy-safe createAction timings. +- Migration: No consumer migration is required; older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 113643f59..13c247ab0 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -60,14 +60,14 @@ authorized release action. | overlays | `@bsv/overlay-discovery-services` | `2.1.6` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | | overlays | `@bsv/overlay-express` | `2.4.9` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | | overlays | `@bsv/overlay-topics` | `1.6.8` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | -| sdk | `@bsv/sdk` | `2.2.18` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | +| sdk | `@bsv/sdk` | `2.3.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.4` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.1.4` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.3` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | | wallet | `@bsv/wallet-relay` | `0.3.4` | cli-library | browser-bundler, browser-esm, cli, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/ts-wallet-relay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ts-wallet-relay) | -| wallet | `@bsv/wallet-toolbox` | `2.4.22` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | -| wallet | `@bsv/wallet-toolbox-client` | `2.4.22` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | -| wallet | `@bsv/wallet-toolbox-mobile` | `2.4.22` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | +| wallet | `@bsv/wallet-toolbox` | `2.5.0` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | +| wallet | `@bsv/wallet-toolbox-client` | `2.5.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | +| wallet | `@bsv/wallet-toolbox-mobile` | `2.5.0` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | ## Standalone infrastructure manifests diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index a541347c6..0633d50da 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "lastReviewed": "2026-07-30", + "lastReviewed": "2026-08-04", "owner": "ts-stack-maintainers", "routineUpdates": { "dependabotConfig": ".github/dependabot.yml", @@ -144,17 +144,17 @@ "closeImplementationWaveAfterPublication": false }, "overrideRemovalReview": { - "reviewedAt": "2026-07-30", - "method": "Regenerated each standalone npm lockfile without gaxios, uuid, or brace-expansion overrides and compared the resulting natural graph; separately rechecked the root Jest/minimatch, typed-rest-client/qs, and isolated Redocly codegen closures.", + "reviewedAt": "2026-08-04", + "method": "Regenerated each standalone npm lockfile without gaxios, uuid, or brace-expansion overrides and compared the resulting natural graph; separately rechecked the root Jest/minimatch, typed-rest-client/qs, and isolated Redocly codegen closures, then refreshed the affected locks against the current brace-expansion, fast-uri, ip-address, and socket.io-parser advisories.", "retainedCount": 19, "result": "Every remaining override still prevents a reproduced vulnerable transitive version or preserves the isolated reproducible codegen closure. No override can be removed safely in this wave.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, or Redocly dependency changes." + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, Redocly, AJV, express-rate-limit, or Socket.IO dependency changes." }, "overrideRegistry": [ { "source": "pnpm-workspace.yaml", - "selector": "brace-expansion@<=5.0.7", - "value": "5.0.8", + "selector": "brace-expansion@<5.0.9", + "value": "5.0.9", "exceptionId": "brace-expansion-jest-override" }, { @@ -176,7 +176,7 @@ "source": "tools/codegen/node/package.json", "selector": "minimatch@10.2.5", "value": { - "brace-expansion": "5.0.8" + "brace-expansion": "5.0.9" }, "exceptionId": "openapi-typescript-redocly-overrides" }, @@ -189,7 +189,7 @@ { "source": "infra/message-box-server/package.json", "selector": "brace-expansion", - "value": "5.0.8", + "value": "5.0.9", "exceptionId": "brace-expansion-jest-override" }, { @@ -213,7 +213,7 @@ { "source": "infra/uhrp-server-basic/package.json", "selector": "brace-expansion", - "value": "5.0.8", + "value": "5.0.9", "exceptionId": "brace-expansion-jest-override" }, { @@ -225,7 +225,7 @@ { "source": "infra/uhrp-server-cloud-bucket/package.json", "selector": "brace-expansion", - "value": "5.0.8", + "value": "5.0.9", "exceptionId": "brace-expansion-jest-override" }, { @@ -255,7 +255,7 @@ { "source": "infra/wab/package.json", "selector": "brace-expansion", - "value": "5.0.8", + "value": "5.0.9", "exceptionId": "brace-expansion-jest-override" }, { diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index e1116418f..6d8c36ee6 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -145,10 +145,10 @@ }, { "name": "@bsv/sdk", - "publishedVersion": "2.2.0", - "releaseType": "patch", - "summary": "Accumulates security and correctness hardening, transaction and action-batch performance work, strict package contracts, safer text and telemetry handling, and caches unchanged BEEF dependency-sort results so repeated wallet known-txid preparation avoids a full topological sort.", - "migration": "No consumer migration is required. BEEF ordering, validation results, serialized bytes, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged." + "publishedVersion": "2.2.18", + "releaseType": "minor", + "summary": "Adds batched proven-transaction BEEF assembly and deferred-then-compound Merkle proof validation, reuses shared proof hashes and BRC-42 counterparty secrets, and accelerates optional-backend P2PKH public-key emission.", + "migration": "No consumer migration is required. The new APIs are additive; BEEF ordering and bytes, proof validity, signatures, synchronous signing, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged." }, { "name": "@bsv/simple", @@ -194,24 +194,24 @@ }, { "name": "@bsv/wallet-toolbox", - "publishedVersion": "2.4.4", - "releaseType": "patch", - "summary": "Adds fee-aware read-only legacy createAction planning, one-transaction funding claims, exact reservation filtering, Knex and IndexedDB funding indexes, faster known-txid and proof handling, and privacy-safe timings for every material funding and BEEF phase.", - "migration": "No consumer migration is required. SQLite/MySQL and IndexedDB indexes are added by the existing migration paths; wallet data and results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged." + "publishedVersion": "2.4.22", + "releaseType": "minor", + "summary": "Makes the successful fragmented createAction path atomic and set-based, overlaps batched proof reads with persistence, batch-validates compound proofs and canonical P2PKH signatures, shares BRC-42 derivation work, removes unused commit reads, bulk-inserts outputs, coalesces authenticated timestamp-only Knex session touches, and adds timings for every remaining material phase.", + "migration": "No consumer migration is required. Storage-provider additions are backward-compatible with fallbacks, existing databases use the normal migration path, and wallet results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged." }, { "name": "@bsv/wallet-toolbox-client", - "publishedVersion": "2.4.4", - "releaseType": "patch", - "summary": "Carries the lockstep browser build with read-only funding planning, atomic claims, IndexedDB funding indexes, BEEF-history caching, and expanded privacy-safe createAction timings.", - "migration": "No consumer migration is required; IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged." + "publishedVersion": "2.4.22", + "releaseType": "minor", + "summary": "Carries the lockstep browser build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, and complete privacy-safe createAction timings.", + "migration": "No consumer migration is required; older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged." }, { "name": "@bsv/wallet-toolbox-mobile", - "publishedVersion": "2.4.4", - "releaseType": "patch", - "summary": "Carries the lockstep mobile build with read-only funding planning, atomic claims, faster BEEF-history handling, and expanded privacy-safe createAction timings.", - "migration": "No consumer migration is required; BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged." + "publishedVersion": "2.4.22", + "releaseType": "minor", + "summary": "Carries the lockstep mobile build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, and complete privacy-safe createAction timings.", + "migration": "No consumer migration is required; older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index 8ac5de719..ab9e026bb 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -320,13 +320,13 @@ "@bsv/overlay-discovery-services": "2.1.6", "@bsv/overlay-express": "2.4.9", "@bsv/overlay-topics": "1.6.8", - "@bsv/sdk": "2.2.18", + "@bsv/sdk": "2.3.0", "@bsv/verifast": "0.3.4", "@bsv/btms": "1.1.4", "@bsv/btms-permission-module": "1.1.3", "@bsv/wallet-relay": "0.3.4", - "@bsv/wallet-toolbox-client": "2.4.22", - "@bsv/wallet-toolbox-mobile": "2.4.22", - "@bsv/wallet-toolbox": "2.4.22" + "@bsv/wallet-toolbox-client": "2.5.0", + "@bsv/wallet-toolbox-mobile": "2.5.0", + "@bsv/wallet-toolbox": "2.5.0" } } diff --git a/governance/repository-health/exceptions.json b/governance/repository-health/exceptions.json index 672d6f3d8..d7de51fa0 100644 --- a/governance/repository-health/exceptions.json +++ b/governance/repository-health/exceptions.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "lastReviewed": "2026-07-30", + "lastReviewed": "2026-08-04", "exceptions": [ { "id": "scorecard-maintainer-merge-policy", @@ -89,7 +89,7 @@ "category": "override", "target": "pnpm-workspace.yaml and four standalone Jest service manifests overriding brace-expansion <=5.0.7", "owner": "ts-stack-maintainers", - "reason": "Jest 30.4.2 still constrains its reporting and coverage paths to minimatch releases that require brace-expansion 1.x/2.x, while GHSA-mh99-v99m-4gvg is fixed only in brace-expansion 5.0.8.", + "reason": "Jest 30.4.2 still constrains reporting and coverage paths to minimatch releases that require older brace-expansion ranges. GHSA-mh99-v99m-4gvg required 5.0.8, and the follow-up GHSA-rgw5-rvv9-x895 requires 5.0.9, so the governed substitution selects the current patched release.", "evidence": [ "pnpm-workspace.yaml#overrides", "pnpm-lock.yaml#overrides", @@ -99,11 +99,12 @@ "infra/uhrp-server-cloud-bucket/package.json#overrides", "infra/wab/package.json#overrides", "https://github.com/advisories/GHSA-mh99-v99m-4gvg", + "https://github.com/advisories/GHSA-rgw5-rvv9-x895", "https://github.com/bsv-blockchain/ts-stack/issues/324" ], "created": "2026-07-25", "reviewBy": "2026-08-25", - "removeWhen": "Remove after the supported Jest dependency graph adopts minimatch 10.2.5 or newer and natively resolves brace-expansion 5.0.8 or newer." + "removeWhen": "Remove after the supported Jest dependency graph natively resolves brace-expansion 5.0.9 or newer across every coverage and reporting path." }, { "id": "standalone-gaxios-advisory-override", @@ -168,6 +169,7 @@ "tools/codegen/node/package-lock.json", "https://github.com/advisories/GHSA-52cp-r559-cp3m", "https://github.com/advisories/GHSA-mh99-v99m-4gvg", + "https://github.com/advisories/GHSA-rgw5-rvv9-x895", "https://github.com/openapi-ts/openapi-typescript/blob/openapi-typescript%407.13.0/packages/openapi-typescript/package.json" ], "created": "2026-07-25", diff --git a/infra/message-box-server/package-lock.json b/infra/message-box-server/package-lock.json index 181dcf332..a11158c93 100644 --- a/infra/message-box-server/package-lock.json +++ b/infra/message-box-server/package-lock.json @@ -5546,9 +5546,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -6625,9 +6625,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -7713,9 +7713,9 @@ } }, "node_modules/ip-address": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.2.tgz", - "integrity": "sha512-aNJVEVdXTr/g5+N2VCLo+CTrFLo/Y+9rx9RC5BpxPtf7NEknmzY+UQvMO8Fjni5KFmDaHJieL8HCMJKjJXsTgg==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", + "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", "license": "MIT", "engines": { "node": ">= 12" @@ -10529,9 +10529,9 @@ } }, "node_modules/socket.io-parser": { - "version": "4.2.6", - "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.6.tgz", - "integrity": "sha512-asJqbVBDsBCJx0pTqw3WfesSY0iRX+2xzWEWzrpcH7L6fLzrhyF8WPI8UaeM4YCuDfpwA/cgsdugMsmtz8EJeg==", + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.7.tgz", + "integrity": "sha512-IH/iSeO9T6gz1KkFleGDWkG9N3dl4jXVYUtMhIqH10Md0ttMer8nUNWiP1DKuNrybD2xBrixLJdCC9J6ECoYkg==", "license": "MIT", "dependencies": { "@socket.io/component-emitter": "~3.1.0", diff --git a/infra/message-box-server/package.json b/infra/message-box-server/package.json index 48912e0af..f2a41532f 100644 --- a/infra/message-box-server/package.json +++ b/infra/message-box-server/package.json @@ -3,7 +3,7 @@ "private": true, "version": "1.1.23", "overrides": { - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "gaxios": "7.3.0", "uuid": "11.1.1" }, diff --git a/infra/uhrp-server-basic/package-lock.json b/infra/uhrp-server-basic/package-lock.json index e2cd2d94b..e37a8e194 100644 --- a/infra/uhrp-server-basic/package-lock.json +++ b/infra/uhrp-server-basic/package-lock.json @@ -4458,9 +4458,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { diff --git a/infra/uhrp-server-basic/package.json b/infra/uhrp-server-basic/package.json index 6f7b7086e..abb529f51 100644 --- a/infra/uhrp-server-basic/package.json +++ b/infra/uhrp-server-basic/package.json @@ -2,7 +2,7 @@ "name": "@bsv/uhrp-lite", "version": "0.1.19", "overrides": { - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "gaxios": "7.3.0" }, "engines": { diff --git a/infra/uhrp-server-cloud-bucket/package-lock.json b/infra/uhrp-server-cloud-bucket/package-lock.json index b9d54ab1b..4f30550dc 100644 --- a/infra/uhrp-server-cloud-bucket/package-lock.json +++ b/infra/uhrp-server-cloud-bucket/package-lock.json @@ -4969,9 +4969,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { @@ -6695,9 +6695,9 @@ } }, "node_modules/ip-address": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.2.tgz", - "integrity": "sha512-aNJVEVdXTr/g5+N2VCLo+CTrFLo/Y+9rx9RC5BpxPtf7NEknmzY+UQvMO8Fjni5KFmDaHJieL8HCMJKjJXsTgg==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", + "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", "license": "MIT", "engines": { "node": ">= 12" diff --git a/infra/uhrp-server-cloud-bucket/package.json b/infra/uhrp-server-cloud-bucket/package.json index 7039eb3aa..4c245daf4 100644 --- a/infra/uhrp-server-cloud-bucket/package.json +++ b/infra/uhrp-server-cloud-bucket/package.json @@ -2,7 +2,7 @@ "name": "@bsv/uhrp-storage-server", "version": "0.2.21", "overrides": { - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "gaxios": "7.3.0", "uuid": "11.1.1" }, diff --git a/infra/wab/package-lock.json b/infra/wab/package-lock.json index 5c71c895e..7e8ba943d 100644 --- a/infra/wab/package-lock.json +++ b/infra/wab/package-lock.json @@ -4648,9 +4648,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { @@ -6279,9 +6279,9 @@ } }, "node_modules/ip-address": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", - "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", + "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", "license": "MIT", "engines": { "node": ">= 12" @@ -7902,16 +7902,6 @@ "node": ">=10" } }, - "node_modules/node-gyp/node_modules/undici": { - "version": "6.27.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.27.0.tgz", - "integrity": "sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=18.17" - } - }, "node_modules/node-gyp/node_modules/which": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", @@ -9863,6 +9853,16 @@ "node": ">=0.8.0" } }, + "node_modules/undici": { + "version": "6.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", diff --git a/infra/wab/package.json b/infra/wab/package.json index 9b88d7538..0f6d57fbf 100644 --- a/infra/wab/package.json +++ b/infra/wab/package.json @@ -3,7 +3,7 @@ "private": true, "version": "1.4.20", "overrides": { - "brace-expansion": "5.0.8", + "brace-expansion": "5.0.9", "gaxios": "7.3.0" }, "engines": { diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 2c46df11c..c11820ebd 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -216,6 +216,13 @@ All notable changes to this project will be documented in this file. The format ### Added +- Add `Beef.mergeProvenTxs` and opt-in deferred `MerklePath` root validation + for storage engines that batch independently proven transactions. The final + compound proof is still validated before use, while shared branches are + hashed once instead of once per transaction. +- Add `KeyDeriver.derivePrivateKeys` so fragmented wallets can reuse one BRC-42 + shared-secret calculation per counterparty while deriving many exact child + keys. Single-key derivation and derived key bytes are unchanged. - Add explicit script-verification context and a distinct resource-exhaustion error so transaction version and local interpreter limits cannot be mistaken for consensus validity. @@ -230,6 +237,11 @@ All notable changes to this project will be documented in this file. The format ### Changed +- Batch BEEF mutation bookkeeping and reuse compound Merkle intermediate hashes. + The optional asynchronous P2PKH backend now forwards its already validated + compressed public key directly into the unlocking script. Existing BEEF + bytes, proof validation, synchronous signing, and transaction signatures are + unchanged. - Cache BEEF dependency-sort results until transaction or proof state changes, so repeated wallet known-txid preparation is proportional to the returned list instead of re-running the complete topological sort. BEEF ordering, diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 7db2cf106..7dceb5e26 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -16,12 +16,12 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 735000, + "raw": 736000, "gzip": 185000, "brotli": 150000 }, "esbuild": { - "raw": 555000, + "raw": 556000, "gzip": 168000, "brotli": 140000 } @@ -30,7 +30,7 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 547000, + "raw": 551000, "gzip": 162000, "brotli": 136000 } diff --git a/packages/sdk/docs/performance.md b/packages/sdk/docs/performance.md index 7c79f50fb..907c3a399 100644 --- a/packages/sdk/docs/performance.md +++ b/packages/sdk/docs/performance.md @@ -1,5 +1,24 @@ # Performance Benchmarks +## Batched proven-transaction BEEF assembly (August 2026) + +Wallets with fragmented funding can require more than one hundred independent +transactions and Merkle proofs in a single Atomic BEEF. `Beef.mergeProvenTxs` +batches mutation synchronization, groups proofs by block/root, validates each +compound proof once, and reuses shared intermediate hashes. The final BEEF is +byte-equivalent to sequential validated merges. + +This optimization is exercised by Wallet Toolbox's retained +`bench:create-action-beef` workload. On a selective production-shaped PXC +database copy, the 110-input direct storage path improved from approximately +1.03 s before batched proof work to 77.8 ms p50 and 151.2 ms p95 across five +samples. Results depend on proof shape, database locality, and host resources. + +`KeyDeriver.derivePrivateKeys` provides the corresponding fragmented-signing +lane: requests are grouped by counterparty, the BRC-42 shared secret is derived +or retrieved once per group, and every invoice-specific HMAC and child scalar +retains the same single-key derivation semantics and bytes. + ## Repeated BEEF dependency sorting (August 2026) Wallet transaction creation can ask the same unchanged BEEF for its dependency diff --git a/packages/sdk/package.json b/packages/sdk/package.json index cf1eee307..e0a59ff3c 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/sdk", - "version": "2.2.18", + "version": "2.3.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/sdk/src/script/templates/P2PKH.ts b/packages/sdk/src/script/templates/P2PKH.ts index f982722ca..10d0ba311 100644 --- a/packages/sdk/src/script/templates/P2PKH.ts +++ b/packages/sdk/src/script/templates/P2PKH.ts @@ -10,7 +10,6 @@ import { sha256 } from '../../primitives/Hash.js' import Script from '../Script.js' import { computeSignatureScope, resolveSourceDetails, formatPreimage } from './SignatureUtils.js' import Signature from '../../primitives/Signature.js' -import PublicKey from '../../primitives/PublicKey.js' import { readyAsyncCryptoBackend, validateAsyncCryptoBytes @@ -95,12 +94,16 @@ export default class P2PKH implements ScriptTemplate { const preimageHash = sha256(preimage) const signingBackend = readyAsyncCryptoBackend('signDigest') + const publicKeyBackend = readyAsyncCryptoBackend('publicKeyFromPrivate') + const privateKeyBytes = signingBackend === undefined && publicKeyBackend === undefined + ? undefined + : Uint8Array.from(privateKey.toArray('be', 32)) const rawSignature = signingBackend === undefined ? privateKey.sign(preimageHash) : Signature.fromDER(Array.from(validateAsyncCryptoBytes( 'signDigest', await signingBackend.signDigest( - Uint8Array.from(privateKey.toArray('be', 32)), + privateKeyBytes!, // PrivateKey.sign hashes its argument before ECDSA signing. // Preserve that historical double-SHA256 contract when passing a // digest to a backend that signs the supplied bytes directly. @@ -113,16 +116,9 @@ export default class P2PKH implements ScriptTemplate { signatureScope ) const sigForScript = sig.toChecksigFormat() - const publicKeyBackend = readyAsyncCryptoBackend('publicKeyFromPrivate') const pubkeyForScript = publicKeyBackend === undefined ? privateKey.toPublicKey().encode(true) as number[] - : PublicKey.fromDER(Array.from(validateAsyncCryptoBytes( - 'publicKeyFromPrivate', - await publicKeyBackend.publicKeyFromPrivate( - Uint8Array.from(privateKey.toArray('be', 32)) - ), - 33 - ))).encode(true) as number[] + : Array.from(validateCompressedPublicKey(await publicKeyBackend.publicKeyFromPrivate(privateKeyBytes!))) return new UnlockingScript([ { op: sigForScript.length, data: sigForScript }, { op: pubkeyForScript.length, data: pubkeyForScript } @@ -136,3 +132,11 @@ export default class P2PKH implements ScriptTemplate { } } } + +function validateCompressedPublicKey (value: Uint8Array): Uint8Array { + const bytes = validateAsyncCryptoBytes('publicKeyFromPrivate', value, 33) + if (bytes[0] !== 0x02 && bytes[0] !== 0x03) { + throw new Error('publicKeyFromPrivate returned an invalid compressed public key') + } + return bytes +} diff --git a/packages/sdk/src/script/templates/__tests/P2PKH.async-backend.test.ts b/packages/sdk/src/script/templates/__tests/P2PKH.async-backend.test.ts new file mode 100644 index 000000000..8a0999c1d --- /dev/null +++ b/packages/sdk/src/script/templates/__tests/P2PKH.async-backend.test.ts @@ -0,0 +1,69 @@ +import { + type AsyncCryptoBackend, + registerAsyncCryptoBackend, + unregisterAsyncCryptoBackend +} from '../../../primitives/AsyncCryptoBackend' +import PrivateKey from '../../../primitives/PrivateKey' +import Script from '../../Script' +import Transaction from '../../../transaction/Transaction' +import P2PKH from '../P2PKH' + +function backendFor (key: PrivateKey, publicKey: Uint8Array): AsyncCryptoBackend { + const derSignature = Uint8Array.from(key.sign(Array.from({ length: 32 }, () => 1)).toDER()) + return { + preload: async () => {}, + isReady: () => true, + supportsCrypto: operation => operation === 'signDigest' || operation === 'publicKeyFromPrivate', + signDigest: async () => derSignature, + verifyDigest: async () => true, + verifyDigestBatch: async items => items.map(() => true), + publicKeyFromPrivate: async () => publicKey, + multiplyPublicKey: async value => value, + tweakPublicKeyAdd: async value => value, + tweakPrivateKeyAdd: async value => value + } +} + +function spendFor (key: PrivateKey): { tx: Transaction; template: ReturnType } { + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromHex('00'), + sequence: 0xffffffff + }) + const p2pkh = new P2PKH() + source.addOutput({ satoshis: 2, lockingScript: p2pkh.lock(key.toAddress()) }) + const tx = new Transaction() + tx.addInput({ sourceTransaction: source, sourceOutputIndex: 0, sequence: 0xffffffff }) + tx.addOutput({ satoshis: 1, lockingScript: Script.fromHex('51') }) + return { tx, template: p2pkh.unlock(key) } +} + +describe('P2PKH async crypto backend', () => { + test('forwards the validated compressed public key without parsing it again', async () => { + const key = PrivateKey.fromRandom() + const publicKey = Uint8Array.from(key.toPublicKey().encode(true) as number[]) + const backend = backendFor(key, publicKey) + registerAsyncCryptoBackend(backend) + try { + const { tx, template } = spendFor(key) + const script = await template.sign(tx, 0) + expect(script.chunks.at(-1)?.data).toEqual(Array.from(publicKey)) + } finally { + unregisterAsyncCryptoBackend(backend) + } + }) + + test('rejects a malformed compressed public key returned by a backend', async () => { + const key = PrivateKey.fromRandom() + const backend = backendFor(key, Uint8Array.from([0x04, ...Array.from({ length: 32 }, () => 0)])) + registerAsyncCryptoBackend(backend) + try { + const { tx, template } = spendFor(key) + await expect(template.sign(tx, 0)).rejects.toThrow('invalid compressed public key') + } finally { + unregisterAsyncCryptoBackend(backend) + } + }) +}) diff --git a/packages/sdk/src/transaction/Beef.ts b/packages/sdk/src/transaction/Beef.ts index 81bb38ae2..3bcb3b2fd 100644 --- a/packages/sdk/src/transaction/Beef.ts +++ b/packages/sdk/src/transaction/Beef.ts @@ -1,4 +1,4 @@ -import MerklePath from './MerklePath.js' +import MerklePath, { type MerklePathLeaf } from './MerklePath.js' import Transaction from './Transaction.js' import ChainTracker from './ChainTracker.js' import BeefTx from './BeefTx.js' @@ -16,6 +16,50 @@ import { hash256 } from '../primitives/Hash.js' import { BEEF_V1, BEEF_V2, ATOMIC_BEEF } from './BeefConstants.js' export { BEEF_V1, BEEF_V2, ATOMIC_BEEF, TX_DATA_FORMAT } from './BeefConstants.js' +function mergeCompatibleBumpLevels( + levels: Array>, + other: MerklePath, + expectedLevels: number, + validateCombined: boolean +): void { + if (other.path.length !== expectedLevels) throw new Error('Mismatched roots') + for (let height = 0; height < expectedLevels; height++) { + mergeCompatibleBumpLevel(levels[height], other.path[height], validateCombined) + } +} + +function mergeCompatibleBumpLevel( + level: Map, + otherLeaves: MerklePathLeaf[], + validateCombined: boolean +): void { + for (const otherLeaf of otherLeaves) { + const existing = level.get(otherLeaf.offset) + if (existing == null) { + level.set(otherLeaf.offset, otherLeaf) + continue + } + mergeCompatibleBumpLeaf(existing, otherLeaf, validateCombined) + } +} + +function mergeCompatibleBumpLeaf( + existing: MerklePathLeaf, + other: MerklePathLeaf, + validateCombined: boolean +): void { + if (validateCombined && (existing.hash !== other.hash || existing.duplicate !== other.duplicate)) { + throw new Error('Mismatched roots') + } + if (other.txid != null) existing.txid = true +} + +function indexBumpTxids(bump: MerklePath, index: number, byTxid: Map): void { + for (const leaf of bump.path[0]) { + if (typeof leaf.hash === 'string') byTxid.set(leaf.hash, index) + } +} + interface BeefTxSerializationState { ref: BeefTx bumpIndex: number | undefined @@ -119,7 +163,7 @@ export class Beef { atomicTxid: string | undefined = undefined private txidIndex: Map | undefined = undefined private txPositionIndex: Map | undefined = undefined - private bumpIndexByKey: Map | undefined = undefined + private bumpIndexesByHeight: Map | undefined = undefined private bumpIndexByTxid: Map | undefined = undefined private rawBytesCache?: Uint8Array private hexCache?: string @@ -545,19 +589,21 @@ export class Beef { return this.bumpIndexByTxid } - private ensureBumpKeyIndex(): Map { - if (this.bumpIndexByKey == null) { - this.bumpIndexByKey = new Map() + private ensureBumpHeightIndex(): Map { + if (this.bumpIndexesByHeight == null) { + this.bumpIndexesByHeight = new Map() for (let i = 0; i < this.bumps.length; i++) { const bump = this.bumps[i] - this.bumpIndexByKey.set(`${bump.blockHeight}:${bump.computeRoot()}`, i) + const indexes = this.bumpIndexesByHeight.get(bump.blockHeight) ?? [] + indexes.push(i) + this.bumpIndexesByHeight.set(bump.blockHeight, indexes) } } - return this.bumpIndexByKey + return this.bumpIndexesByHeight } private invalidateBumpIndexes(): void { - this.bumpIndexByKey = undefined + this.bumpIndexesByHeight = undefined this.bumpIndexByTxid = undefined } @@ -646,6 +692,84 @@ export class Beef { this.synchronizeNestedTransactionMutations() this.synchronizeNestedBumpMutations() this.markMutated(false) + return this.mergeBumpEntry(bump) + } + + /** + * Merge several independently proven transactions in one mutation pass. + * + * This is equivalent to calling `mergeRawTx` followed by `mergeBump` for + * every entry, but synchronizes nested BEEF state only once. That distinction + * matters for wallets assembling a BEEF from a fragmented UTXO set because + * proof paths are otherwise re-scanned after every input. + */ + mergeProvenTxs(entries: Array<{ + rawTx: number[] | Uint8Array + merklePath: MerklePath + merkleRoot?: string + }>): BeefTx[] { + if (entries.length === 0) return [] + this.synchronizeNestedTransactionMutations() + this.synchronizeNestedBumpMutations() + this.markMutated(true) + const merged: BeefTx[] = [] + for (const entry of entries) { + merged.push(this.mergeRawTxEntry(entry.rawTx)) + } + + // A real wallet often owns many transactions mined in the same block. + // Combine those paths once per block root, rather than repeatedly hashing, + // combining, and trimming an ever-growing compound proof. + const heightCounts = new Map() + for (const entry of entries) { + const height = entry.merklePath.blockHeight + heightCounts.set(height, (heightCounts.get(height) ?? 0) + 1) + } + const groups = new Map() + for (let index = 0; index < entries.length; index++) { + const path = entries[index].merklePath + const rootHint = entries[index].merkleRoot + const key = heightCounts.get(path.blockHeight) === 1 + ? `${path.blockHeight}:single:${index}` + : `${path.blockHeight}:${rootHint ?? path.computeRoot()}` + const group = groups.get(key) + if (group == null) { + groups.set(key, { first: index, paths: [path], validateCombined: rootHint != null }) + } else { + group.paths.push(path) + group.validateCombined = group.validateCombined || rootHint != null + } + } + for (const group of [...groups.values()].sort((a, b) => a.first - b.first)) { + this.mergeBumpEntry(this.combineCompatibleBumps(group.paths, group.validateCombined)) + } + return merged + } + + /** Combine already root-matched paths while preserving the first path reference. */ + private combineCompatibleBumps(paths: MerklePath[], validateCombined: boolean = false): MerklePath { + const combined = paths[0] + if (paths.length === 1 && !validateCombined) return combined + const levels = combined.path.map(level => new Map(level.map(leaf => [leaf.offset, leaf]))) + for (let pathIndex = 1; pathIndex < paths.length; pathIndex++) { + mergeCompatibleBumpLevels(levels, paths[pathIndex], combined.path.length, validateCombined) + } + const combinedPath = levels.map(level => [...level.values()]) + if (validateCombined) { + // Storage-backed paths may have intentionally deferred root validation. + // Validate the compound path once here so shared branches are hashed + // once, while corrupt or mismatched proofs still fail closed. + const validated = new MerklePath(combined.blockHeight, combinedPath) + validated.trim() + return validated + } + combined.path = combinedPath + combined.trim() + return combined + } + + /** Merge one bump after the caller has synchronized and marked the BEEF. */ + private mergeBumpEntry(bump: MerklePath): number { const bumpIndex = this.findOrInsertBump(bump) const b = this.bumps[bumpIndex] @@ -663,23 +787,23 @@ export class Beef { * Find an existing compatible bump or insert a new one; return its index. */ private findOrInsertBump(bump: MerklePath): number { - const byKey = this.ensureBumpKeyIndex() + const byHeight = this.ensureBumpHeightIndex() const byTxid = this.ensureBumpTxidIndex() - const key = `${bump.blockHeight}:${bump.computeRoot()}` - const existing = byKey.get(key) - if (existing !== undefined) { - this.bumps[existing].combine(bump) - for (const leaf of this.bumps[existing].path[0]) { - if (typeof leaf.hash === 'string') byTxid.set(leaf.hash, existing) + const sameHeight = byHeight.get(bump.blockHeight) ?? [] + if (sameHeight.length > 0) { + const root = bump.computeRoot() + for (const existing of sameHeight) { + if (this.bumps[existing].computeRoot() !== root) continue + this.bumps[existing].combine(bump) + indexBumpTxids(this.bumps[existing], existing, byTxid) + return existing } - return existing } this.bumps.push(bump) const index = this.bumps.length - 1 - byKey.set(key, index) - for (const leaf of bump.path[0]) { - if (typeof leaf.hash === 'string') byTxid.set(leaf.hash, index) - } + sameHeight.push(index) + byHeight.set(bump.blockHeight, sameHeight) + indexBumpTxids(bump, index, byTxid) return index } @@ -709,6 +833,11 @@ export class Beef { mergeRawTx(rawTx: number[] | Uint8Array, bumpIndex?: number): BeefTx { this.synchronizeNestedTransactionMutations() this.markMutated(true) + return this.mergeRawTxEntry(rawTx, bumpIndex) + } + + /** Merge one raw transaction after the caller has synchronized and marked the BEEF. */ + private mergeRawTxEntry(rawTx: number[] | Uint8Array, bumpIndex?: number): BeefTx { const newTx: BeefTx = new BeefTx(rawTx, bumpIndex) this.replaceOrAppendTx(newTx) this.tryToValidateBumpIndex(newTx) @@ -716,7 +845,7 @@ export class Beef { } private mergeTransactionEntry(current: Transaction): BeefTx { - const bumpIndex = current.merklePath == null ? undefined : this.mergeBump(current.merklePath) + const bumpIndex = current.merklePath == null ? undefined : this.mergeBumpEntry(current.merklePath) const newTx = new BeefTx(current, bumpIndex) this.replaceOrAppendTx(newTx) this.tryToValidateBumpIndex(newTx) @@ -743,6 +872,11 @@ export class Beef { mergeTransaction(tx: Transaction): BeefTx { this.synchronizeNestedTransactionMutations() this.markMutated(true) + return this.mergeTransactionGraph(tx) + } + + /** Merge one transaction graph after the caller has synchronized and marked the BEEF. */ + private mergeTransactionGraph(tx: Transaction): BeefTx { tx.materializeSourceTXIDs() const rootTxid = tx.id('hex') const visited = new Set() @@ -809,15 +943,40 @@ export class Beef { return beefTx } + /** Merge one BEEF transaction after the caller has synchronized and marked the BEEF. */ + private mergeBeefTxEntry(btx: BeefTx): BeefTx { + let beefTx = this.findTxidIndexed(btx.txid) + + if (btx.isTxidOnly && beefTx == null) { + beefTx = BeefTx.fromTxid(btx.txid) + this.txs.push(beefTx) + this.addToIndex(beefTx) + this.tryToValidateBumpIndex(beefTx) + } else if (btx._tx != null && (beefTx == null || beefTx.isTxidOnly)) { + beefTx = this.mergeTransactionGraph(btx._tx) + } else if (btx._rawTx != null && (beefTx == null || beefTx.isTxidOnly)) { + beefTx = this.mergeRawTxEntry(btx._rawTx) + } + + if (beefTx == null) { + throw new Error(`Failed to merge BeefTx for txid: ${btx.txid}`) + } + + return beefTx + } + mergeBeef(beef: Beef | number[] | Uint8Array): void { const b: Beef = beef instanceof Beef ? beef : Beef.fromBinary(beef) + this.synchronizeNestedTransactionMutations() + this.synchronizeNestedBumpMutations() + this.markMutated(true) for (const bump of b.bumps) { - this.mergeBump(bump) + this.mergeBumpEntry(bump) } for (const tx of b.txs) { - this.mergeBeefTx(tx) + this.mergeBeefTxEntry(tx) } } @@ -1434,7 +1593,7 @@ export class Beef { c.txs = Array.from(this.txs) c.txidIndex = undefined c.txPositionIndex = undefined - c.bumpIndexByKey = undefined + c.bumpIndexesByHeight = undefined c.bumpIndexByTxid = undefined c.needsSort = this.needsSort c.sortResultCache = this.sortResultCache == null ? undefined : this.cloneSortResult(this.sortResultCache) diff --git a/packages/sdk/src/transaction/BeefParty.ts b/packages/sdk/src/transaction/BeefParty.ts index 7e3358f8a..cb61141e3 100644 --- a/packages/sdk/src/transaction/BeefParty.ts +++ b/packages/sdk/src/transaction/BeefParty.ts @@ -120,7 +120,12 @@ export class BeefParty extends Beef { const b: Beef = beef instanceof Beef ? beef : Beef.fromBinary(beef) const knownTxids = b.getValidTxids() this.mergeBeef(b) - this.addKnownTxidsForParty(party, knownTxids) + if (!this.isParty(party)) this.addParty(party) + const knownToParty = this.knownTo[party] + // Every valid txid was just bulk-merged above. Recording party knowledge + // directly avoids re-running the BEEF-wide mutation synchronization once + // per txid through mergeTxidOnly. + for (const txid of knownTxids) knownToParty[txid] = true } } diff --git a/packages/sdk/src/transaction/MerklePath.ts b/packages/sdk/src/transaction/MerklePath.ts index 080f00cd6..ee3fd9a4e 100644 --- a/packages/sdk/src/transaction/MerklePath.ts +++ b/packages/sdk/src/transaction/MerklePath.ts @@ -10,6 +10,41 @@ export interface MerklePathLeaf { duplicate?: boolean } +function hashPair(left: string | undefined, right: string | undefined): string { + return toHex(hash256(toArray((left ?? '') + (right ?? ''), 'hex').reverse()).reverse()) +} + +function cachedMerkleRoot( + nodeKey: string, + workingHash: string, + treeHeight: number, + nodeHashCache: Map +): string | undefined { + const cachedNodeHash = nodeHashCache.get(nodeKey) + if (cachedNodeHash == null) return undefined + if (cachedNodeHash !== workingHash) throw new Error('Mismatched roots') + const root = nodeHashCache.get(`${treeHeight}:0`) + if (root == null) throw new Error('Mismatched roots') + return root +} + +function nextCachedHash( + workingHash: string, + leaf: MerklePathLeaf | undefined, + offset: number, + index: number, + height: number, + isLastOddNode: boolean +): string { + if (leaf == null) { + if (isLastOddNode) return hashPair(workingHash, workingHash) + throw new Error(`Missing hash for index ${index} at height ${height}`) + } + if (leaf.duplicate === true) return hashPair(workingHash, workingHash) + if (offset % 2 === 1) return hashPair(leaf.hash, workingHash) + return hashPair(workingHash, leaf.hash) +} + /** * Represents a Merkle Path, which is used to provide a compact proof of inclusion for a * transaction in a block. This class encapsulates all the details required for creating @@ -55,7 +90,8 @@ export default class MerklePath { static fromReader( reader: Reader | ReaderUint8Array, - legalOffsetsOnly: boolean = true + legalOffsetsOnly: boolean = true, + validateRoots: boolean = true ): MerklePath { const blockHeight = reader.readVarIntNum() const treeHeight = reader.readUInt8() @@ -95,7 +131,7 @@ export default class MerklePath { // Sort the array based on the offset property path[level].sort((a, b) => a.offset - b.offset) } - return new MerklePath(blockHeight, path, legalOffsetsOnly) + return new MerklePath(blockHeight, path, legalOffsetsOnly, validateRoots) } /** @@ -105,9 +141,13 @@ export default class MerklePath { * @param {number[]} bump - The binary array representation of the Merkle Path. * @returns {MerklePath} - A new MerklePath instance. */ - static fromBinary(bump: number[] | Uint8Array): MerklePath { + static fromBinary( + bump: number[] | Uint8Array, + legalOffsetsOnly: boolean = true, + validateRoots: boolean = true + ): MerklePath { const reader = new ReaderUint8Array(bump) - return MerklePath.fromReader(reader) + return MerklePath.fromReader(reader, legalOffsetsOnly, validateRoots) } /** @@ -135,7 +175,8 @@ export default class MerklePath { duplicate?: boolean }> >, - legalOffsetsOnly: boolean = true + legalOffsetsOnly: boolean = true, + validateRoots: boolean = true ) { this.blockHeight = blockHeight this.path = path @@ -168,11 +209,29 @@ export default class MerklePath { }) }) - // every txid must calculate to the same root. + // Batch consumers can defer cryptographic root validation until several + // paths from the same block have been combined. The resulting compound + // path must still be constructed with validation enabled before use. + if (!validateRoots) return + + // Every level-zero leaf must calculate to the same root. Share computed + // intermediate nodes across those checks; compound proofs otherwise + // re-hash the same branches once per included transaction. + const sourceIndex = this.path.map(level => new Map(level.map(leaf => [leaf.offset, leaf]))) + const hashCache = new Map() + const nodeHashCache = new Map() + const maxOffset = this.path[0].reduce((max, leaf) => Math.max(max, leaf.offset), 0) let root: string this.path[0].forEach((leaf, idx) => { - if (idx === 0) root = this.computeRoot(leaf.hash) - if (root !== this.computeRoot(leaf.hash)) { + const computed = this.computeRootCached( + leaf.hash, + sourceIndex, + hashCache, + nodeHashCache, + maxOffset + ) + if (idx === 0) root = computed + if (root !== computed) { throw new Error('Mismatched roots') } }) @@ -247,8 +306,37 @@ export default class MerklePath { return leaf.offset } - private static hashPair(left: string | undefined, right: string | undefined): string { - return toHex(hash256(toArray((left ?? '') + (right ?? ''), 'hex').reverse()).reverse()) + private computeRootCached( + txid: string | undefined, + sourceIndex: Array>, + hashCache: Map, + nodeHashCache: Map, + maxOffset: number + ): string { + if (typeof txid !== 'string') txid = this.path[0].find(leaf => Boolean(leaf.hash))?.hash + if (typeof txid !== 'string') throw new TypeError('Transaction ID is undefined') + const index = this.indexOf(txid) + if (this.path.length === 1 && this.path[0].length === 1) return txid + const treeHeight = Math.max(this.path.length, 32 - Math.clz32(maxOffset)) + let workingHash = txid + for (let height = 0; height < treeHeight; height++) { + const nodeKey = `${height}:${index >> height}` + const cachedRoot = cachedMerkleRoot(nodeKey, workingHash, treeHeight, nodeHashCache) + if (cachedRoot != null) return cachedRoot + nodeHashCache.set(nodeKey, workingHash) + const offset = (index >> height) ^ 1 + const leaf = this.cachedFindLeaf(height, offset, sourceIndex, hashCache, maxOffset) + workingHash = nextCachedHash( + workingHash, + leaf, + offset, + index, + height, + this.path.length === 1 && index >> height === maxOffset >> height + ) + } + nodeHashCache.set(`${treeHeight}:0`, workingHash) + return workingHash } private nextRootHash( @@ -261,13 +349,13 @@ export default class MerklePath { const leaf = this.findOrComputeLeaf(height, offset) if (leaf == null) { const isLastOddNode = this.path.length === 1 && index >> height === maxOffset >> height - if (isLastOddNode) return MerklePath.hashPair(workingHash, workingHash) + if (isLastOddNode) return hashPair(workingHash, workingHash) throw new Error(`Missing hash for index ${index} at height ${height}`) } - if (leaf.duplicate === true) return MerklePath.hashPair(workingHash, workingHash) + if (leaf.duplicate === true) return hashPair(workingHash, workingHash) return offset % 2 === 1 - ? MerklePath.hashPair(leaf.hash, workingHash) - : MerklePath.hashPair(workingHash, leaf.hash) + ? hashPair(leaf.hash, workingHash) + : hashPair(workingHash, leaf.hash) } /** diff --git a/packages/sdk/src/transaction/__tests/Beef.test.ts b/packages/sdk/src/transaction/__tests/Beef.test.ts index 7283e6cf9..8fb59506a 100644 --- a/packages/sdk/src/transaction/__tests/Beef.test.ts +++ b/packages/sdk/src/transaction/__tests/Beef.test.ts @@ -4,6 +4,7 @@ import { Beef, BEEF_V1, BEEF_V2 } from '../../transaction/Beef' import Transaction from '../../transaction/Transaction' import { fromBase58 } from '../../primitives/utils' import Script from '../../script/Script' +import MerklePath from '../../transaction/MerklePath' // The following imports allow full type checking by the VsCode editor, but tests will fail to run: /* @@ -171,6 +172,97 @@ describe('Beef tests', () => { expect(beef.isValid(true)).toBe(true) }) + test('mergeProvenTxs is byte-equivalent to sequential proven merges', () => { + const transactions = Array.from({ length: 64 }, (_, index) => { + const transaction = new Transaction() + transaction.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromHex(`04${index.toString(16).padStart(8, '0')}`), + sequence: 0xffffffff + }) + transaction.addOutput({ satoshis: index + 1, lockingScript: Script.fromHex('51') }) + return transaction + }) + const sameBlockTxids = transactions.slice(0, 4).map(transaction => transaction.id('hex')) + const entries = transactions.map((transaction, index) => { + const txid = transaction.id('hex') + const merklePath = index < sameBlockTxids.length + ? new MerklePath(900_000, [sameBlockTxids.map((hash, offset) => ({ + offset, + hash, + txid: offset === index + }))]) + : new MerklePath(900_000 + index, [[{ offset: 0, hash: txid, txid: true }]]) + return { rawTx: transaction.toBinary(), merklePath } + }) + const sequential = new Beef() + for (const entry of entries) { + sequential.mergeRawTx(entry.rawTx) + sequential.mergeBump(entry.merklePath) + } + const batched = new Beef() + const merged = batched.mergeProvenTxs(entries) + + expect(merged.map(tx => tx.txid)).toEqual(sequential.txs.map(tx => tx.txid)) + expect(batched.toHex()).toBe(sequential.toHex()) + expect(batched.isValid()).toBe(true) + }) + + test('mergeProvenTxs validates deferred same-block paths once as a compound proof', () => { + const transactions = Array.from({ length: 8 }, (_, index) => { + const transaction = new Transaction() + transaction.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromHex(`04${(100 + index).toString(16).padStart(8, '0')}`), + sequence: 0xffffffff + }) + transaction.addOutput({ satoshis: index + 1, lockingScript: Script.fromHex('51') }) + return transaction + }) + const txids = transactions.map(transaction => transaction.id('hex')) + const fullPath = new MerklePath(900_100, [ + txids.map((hash, offset) => ({ offset, hash, txid: true })) + ]) + const merkleRoot = fullPath.computeRoot() + const binaries = txids.map(txid => fullPath.extract([txid]).toBinary()) + + const sequential = new Beef() + for (let index = 0; index < transactions.length; index++) { + sequential.mergeRawTx(transactions[index].toBinary()) + sequential.mergeBump(MerklePath.fromBinary(binaries[index])) + } + + const batched = new Beef() + batched.mergeProvenTxs(transactions.map((transaction, index) => ({ + rawTx: transaction.toBinary(), + merklePath: MerklePath.fromBinary(binaries[index], true, false), + merkleRoot + }))) + + expect(batched.toHex()).toBe(sequential.toHex()) + expect(batched.isValid()).toBe(true) + + const corrupt = binaries.map(binary => MerklePath.fromBinary(binary, true, false)) + const conflicting = corrupt[1].path.flat().find(leaf => leaf.txid !== true && leaf.hash != null) + expect(conflicting).toBeDefined() + conflicting!.hash = 'ff'.repeat(32) + expect(() => new Beef().mergeProvenTxs(transactions.map((transaction, index) => ({ + rawTx: transaction.toBinary(), + merklePath: corrupt[index], + merkleRoot + })))).toThrow('Mismatched roots') + + const mismatchedLength = binaries.map(binary => MerklePath.fromBinary(binary, true, false)) + mismatchedLength[1].path.push([]) + expect(() => new Beef().mergeProvenTxs(transactions.map((transaction, index) => ({ + rawTx: transaction.toBinary(), + merklePath: mismatchedLength[index], + merkleRoot + })))).toThrow('Mismatched roots') + }) + test('findTransactionForSigning returns undefined without transaction bytes', () => { const beef = new Beef() const missingTxid = '11'.repeat(32) diff --git a/packages/sdk/src/transaction/__tests/BeefParty.additional.test.ts b/packages/sdk/src/transaction/__tests/BeefParty.additional.test.ts index 3d1a950f8..97a27e5bd 100644 --- a/packages/sdk/src/transaction/__tests/BeefParty.additional.test.ts +++ b/packages/sdk/src/transaction/__tests/BeefParty.additional.test.ts @@ -1,5 +1,21 @@ import BeefParty from '../BeefParty' import { Beef } from '../Beef' +import Transaction from '../Transaction' +import MerklePath from '../MerklePath' +import Script from '../../script/Script' + +function provenTransaction(index: number): Transaction { + const tx = new Transaction() + tx.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromASM(`OP_${(index % 16) + 1}`) + }) + tx.addOutput({ satoshis: index + 1, lockingScript: Script.fromASM('OP_TRUE') }) + const txid = tx.id('hex') + tx.merklePath = new MerklePath(800_000 + index, [[{ offset: 0, hash: txid, txid: true }]]) + return tx +} describe('BeefParty – additional coverage', () => { describe('mergeBeefFromParty', () => { @@ -18,5 +34,20 @@ describe('BeefParty – additional coverage', () => { bp.mergeBeefFromParty('bob', binary) expect(bp.isParty('bob')).toBe(true) }) + + it('synchronizes once for a bulk party merge and records known txids directly', () => { + const incoming = new Beef() + for (let index = 0; index < 64; index++) incoming.mergeTransaction(provenTransaction(index)) + expect(incoming.getValidTxids()).toHaveLength(64) + + const receiver = new BeefParty(['storage']) + const synchronize = jest.spyOn(receiver as any, 'synchronizeNestedTransactionMutations') + receiver.mergeBeefFromParty('storage', incoming) + + expect(synchronize).toHaveBeenCalledTimes(1) + expect(receiver.txs).toHaveLength(64) + expect(receiver.getKnownTxidsForParty('storage')).toHaveLength(64) + expect(receiver.isValid()).toBe(true) + }) }) }) diff --git a/packages/sdk/src/wallet/CachedKeyDeriver.ts b/packages/sdk/src/wallet/CachedKeyDeriver.ts index 02e88182e..d4d4b4973 100644 --- a/packages/sdk/src/wallet/CachedKeyDeriver.ts +++ b/packages/sdk/src/wallet/CachedKeyDeriver.ts @@ -1,5 +1,5 @@ import { Point, PrivateKey, PublicKey, SymmetricKey } from '../primitives/index.js' -import { Counterparty, KeyDeriver, KeyDeriverApi } from './KeyDeriver.js' +import { Counterparty, KeyDeriver, KeyDeriverApi, PrivateKeyDerivation } from './KeyDeriver.js' import { WalletProtocol } from './Wallet.interfaces.js' type CachedKeyValue = PublicKey | PrivateKey | SymmetricKey | Point | number[] @@ -139,6 +139,38 @@ export default class CachedKeyDeriver implements KeyDeriverApi { } } + derivePrivateKeys(derivations: readonly PrivateKeyDerivation[]): PrivateKey[] { + const results: PrivateKey[] = [] + const missing: PrivateKeyDerivation[] = [] + const missingIndexes: number[] = [] + const missingCacheKeys: string[] = [] + for (let index = 0; index < derivations.length; index++) { + const derivation = derivations[index] + const cacheKey = this.generateCacheKey( + 'derivePrivateKey', + derivation.protocolID, + derivation.keyID, + derivation.counterparty + ) + const cached = this.cacheGet(cacheKey) + if (cached instanceof PrivateKey) { + results[index] = cached + } else { + missing.push(derivation) + missingIndexes.push(index) + missingCacheKeys.push(cacheKey) + } + } + if (missing.length > 0) { + const derived = this.keyDeriver.derivePrivateKeys(missing) + for (let index = 0; index < derived.length; index++) { + results[missingIndexes[index]] = derived[index] + this.cacheSet(missingCacheKeys[index], derived[index]) + } + } + return results + } + /** * Derives a symmetric key based on protocol ID, key ID, and counterparty. * Caches the result for future calls with the same parameters. diff --git a/packages/sdk/src/wallet/KeyDeriver.ts b/packages/sdk/src/wallet/KeyDeriver.ts index 7e3085e44..cf6b46180 100644 --- a/packages/sdk/src/wallet/KeyDeriver.ts +++ b/packages/sdk/src/wallet/KeyDeriver.ts @@ -15,6 +15,12 @@ import { WalletProtocol, PubKeyHex } from './Wallet.interfaces.js' export type Counterparty = PublicKey | PubKeyHex +export interface PrivateKeyDerivation { + protocolID: WalletProtocol + keyID: string + counterparty: Counterparty +} + export interface KeyDeriverApi { /** * The root key from which all other keys are derived. @@ -54,6 +60,13 @@ export interface KeyDeriverApi { counterparty: Counterparty ) => PrivateKey + /** + * Derives several private keys while sharing each counterparty ECDH result. + * Implementations that do not provide this additive lane retain the + * per-key `derivePrivateKey` contract. + */ + derivePrivateKeys?: (derivations: readonly PrivateKeyDerivation[]) => PrivateKey[] + /** * Derives a symmetric key based on protocol ID, key ID, and counterparty. * Note: Symmetric keys should not be derivable by everyone due to security risks. @@ -188,6 +201,46 @@ export class KeyDeriver implements KeyDeriverApi { ) } + derivePrivateKeys (derivations: readonly PrivateKeyDerivation[]): PrivateKey[] { + const prepared = derivations.map((derivation, index) => ({ + index, + counterparty: this.normalizeCounterparty(derivation.counterparty), + invoiceNumber: this.computeInvoiceNumber(derivation.protocolID, derivation.keyID) + })) + const groups = new Map() + for (const derivation of prepared) { + const key = derivation.counterparty.toString() + const group = groups.get(key) ?? [] + group.push(derivation) + groups.set(key, group) + } + + const results: PrivateKey[] = [] + for (const group of groups.values()) { + let sharedSecret: Point | undefined + for (let index = 0; index < group.length; index++) { + const derivation = group[index] + results[derivation.index] = this.rootKey.deriveChild( + derivation.counterparty, + derivation.invoiceNumber, + index === 0 + ? (priv, pub, point) => { + sharedSecret = point + this.cacheSharedSecret?.(priv, pub, point) + } + : undefined, + index === 0 + ? (priv, pub) => { + sharedSecret = this.retrieveCachedSharedSecret?.(priv, pub) + return sharedSecret + } + : () => sharedSecret + ) + } + } + return results + } + /** * Derives a symmetric key based on protocol ID, key ID, and counterparty. * Note: Symmetric keys should not be derivable by everyone due to security risks. diff --git a/packages/sdk/src/wallet/Wallet.interfaces.ts b/packages/sdk/src/wallet/Wallet.interfaces.ts index d9d2de3e6..8d40c90ee 100644 --- a/packages/sdk/src/wallet/Wallet.interfaces.ts +++ b/packages/sdk/src/wallet/Wallet.interfaces.ts @@ -438,7 +438,7 @@ export interface SignActionResult { } /** - * @param {Base64String} reference - Reference number for the transaction to abort. + * @param {Base64String} reference - Reference number or transaction ID for the action to abort. */ export interface AbortActionArgs { reference: Base64String diff --git a/packages/sdk/src/wallet/__tests/CachedKeyDeriver.test.ts b/packages/sdk/src/wallet/__tests/CachedKeyDeriver.test.ts index 6739bdc32..042996a08 100644 --- a/packages/sdk/src/wallet/__tests/CachedKeyDeriver.test.ts +++ b/packages/sdk/src/wallet/__tests/CachedKeyDeriver.test.ts @@ -141,6 +141,34 @@ describe('CachedKeyDeriver', () => { }) }) + describe('derivePrivateKeys', () => { + it('derives only cache misses and preserves request order', () => { + const protocolID: [1, string] = [1, 'testprotocol'] + const first = { protocolID, keyID: 'one', counterparty: 'anyone' as const } + const second = { protocolID, keyID: 'two', counterparty: 'anyone' as const } + const firstKey = new PrivateKey(11) + const secondKey = new PrivateKey(12) + const thirdKey = new PrivateKey(13) + mockKeyDeriver.derivePrivateKeys = jest.fn() + .mockReturnValueOnce([firstKey, secondKey]) + .mockReturnValueOnce([thirdKey]) + + const initial = cachedKeyDeriver.derivePrivateKeys([first, second]) + expect(initial[0]).toBe(firstKey) + expect(initial[1]).toBe(secondKey) + const reordered = cachedKeyDeriver.derivePrivateKeys([second, first]) + expect(reordered[0]).toBe(secondKey) + expect(reordered[1]).toBe(firstKey) + expect(mockKeyDeriver.derivePrivateKeys).toHaveBeenCalledTimes(1) + + const third = { protocolID, keyID: 'three', counterparty: 'anyone' as const } + const partiallyCached = cachedKeyDeriver.derivePrivateKeys([first, third]) + expect(partiallyCached[0]).toBe(firstKey) + expect(partiallyCached[1]).toBe(thirdKey) + expect(mockKeyDeriver.derivePrivateKeys).toHaveBeenLastCalledWith([third]) + }) + }) + describe('deriveSymmetricKey', () => { it('should call deriveSymmetricKey on KeyDeriver and cache the result', () => { const protocolID: [2, string] = [2, 'testprotocol'] diff --git a/packages/sdk/src/wallet/__tests/KeyDeriver.test.ts b/packages/sdk/src/wallet/__tests/KeyDeriver.test.ts index 36e3dc813..359c67122 100644 --- a/packages/sdk/src/wallet/__tests/KeyDeriver.test.ts +++ b/packages/sdk/src/wallet/__tests/KeyDeriver.test.ts @@ -118,6 +118,28 @@ describe('KeyDeriver', () => { ) }) + test('batch private-key derivation is byte-equivalent and shares ECDH by counterparty', () => { + const otherCounterparty = new PrivateKey(70).toPublicKey() + const cacheSharedSecret = jest.fn() + const retrieveCachedSharedSecret = jest.fn(() => undefined) + const batched = new KeyDeriver(rootPrivateKey, cacheSharedSecret, retrieveCachedSharedSecret) + const derivations = [ + { protocolID, keyID: 'one', counterparty: counterpartyPublicKey }, + { protocolID, keyID: 'two', counterparty: counterpartyPublicKey.toString() }, + { protocolID, keyID: 'three', counterparty: otherCounterparty } + ] as const + + expect(batched.derivePrivateKeys(derivations).map(key => key.toHex())).toEqual( + derivations.map(derivation => keyDeriver.derivePrivateKey( + derivation.protocolID, + derivation.keyID, + derivation.counterparty + ).toHex()) + ) + expect(retrieveCachedSharedSecret).toHaveBeenCalledTimes(2) + expect(cacheSharedSecret).toHaveBeenCalledTimes(2) + }) + test('should derive the correct symmetric key', () => { const derivedSymmetricKey = keyDeriver.deriveSymmetricKey( protocolID, diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index b6665e6ee..163ce8af0 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -6,6 +6,34 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox (unreleased) +- Preserve the default automatically negotiated in-memory `noSend` batching and + `sendWith` lifecycle introduced in #289. Expand inherited txid-only proof + ancestors for cold clients, preserve caller-declared known txids, report the + actual viable-change funding shortfall, and document aborting listed staged + actions by transaction ID. +- Collapse fragmented `createAction` storage work into one atomic write + transaction, batch proof retrieval and compound-proof validation, overlap + independent proof reads with persistence, bulk-insert untagged outputs, and + batch canonical P2PKH verification. The retained 153-input authenticated + remote PXC workload is below 500 ms at p95. +- Reuse parsed root and counterparty keys and one BRC-42 shared secret per + counterparty while signing managed BRC-29 inputs, keep funding selection and + transaction-size accounting linear, and avoid redundant BEEF validation, + unused process reads, and unconditional commission reads on the successful + path. Generated scripts, signatures, fees, input ordering, BEEF bytes, and + error diagnostics remain equivalence-tested. +- Add bounded-cardinality spans for proof decode/merge, persistence, signing, + verification, result assembly, serialization, and server-side processing. + No telemetry header or protocol field is added; BRC-103/104, AuthFetch, Auth + Express Middleware, AuthSocket, JSON-RPC, and wallet wire behavior are unchanged. +- Coalesce only recent timestamp-only touches for already-authenticated shared + Knex sessions, while immediately persisting every authentication and + certificate transition. This removes a synchronous replicated PXC write from + the normal RPC path; exact per-request persistence remains available with + `touchIntervalMs: 0`. +- Make expired action-batch reservations non-blocking in indexed queries and + repair MySQL rollback support indexes. Existing databases migrate through the + normal Knex path, and full PXC down-to-empty/re-upgrade is regression-tested. - Plan legacy `createAction` funding against the exact unreserved managed-change set before persistence, claim the selected inputs atomically in one storage transaction, and fail economically impossible fragmented wallets before diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index e377a8ab3..8cc54d9a9 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -105,6 +105,7 @@ The retained fragmented-funding benchmark is runnable with: ```bash pnpm bench:create-action-funding +pnpm bench:create-action-beef ``` Against unmodified commit `c212b5ee7`, a representative 102-input SQLite plan @@ -112,6 +113,19 @@ fell from 622 queries, 102 database transactions, and 107.3 ms to 17 queries, one transaction, and 8.8 ms. Query and transaction counts remain flat when the selected input count grows; networked database deployments should benefit most. +The proof-bearing benchmark also exercises the authenticated remote wallet, +real BRC-103 storage RPC, BRC-29 signing, packed WASM digest verification, and +24-level proofs grouped by block. On the PXC staging topology, 20 independent +153-input samples measured 376.0 ms p50 and 461.6 ms p95; the corresponding +direct storage cohort measured 99.3 ms p50 and 137.4 ms p95. A normal one-input +authenticated cohort measured 78.6 ms p50 and 105.6 ms p95. All 3,080 signature +verdicts passed. A selective production-shaped database copy with 110 fragmented +inputs measured 75.5 ms p50 and 155.4 ms p95 for direct storage. The benchmark +captures client, server HTTP, authentication, RPC, storage, signing, +verification, and serialization spans and retains gates of 100 ms p50 / 150 ms +p95 for the normal cohort and 500 ms p95 for the 153-input cohort. These are +regression gates, not universal hardware guarantees. + Trace context remains local to the telemetry carrier and sink. Wallet Toolbox does not add telemetry headers to AuthFetch, so BRC-103/104, Auth Express Middleware, AuthSocket, JSON-RPC, and mixed-version remote storage behavior are @@ -133,7 +147,9 @@ await storage.migrate(storageName, storageIdentityKey) await storage.makeAvailable() const sessionManager = new KnexSessionManager(storage.knex, { - ttlMs: 24 * 60 * 60 * 1000 + ttlMs: 24 * 60 * 60 * 1000, + // Optional. Set to 0 when every authenticated use must update the row. + touchIntervalMs: 60 * 1000 }) const server = new StorageServer(storage, { @@ -158,6 +174,13 @@ const server = new StorageServer(storage, { server.start() ``` +Shared Knex sessions immediately persist authentication, nonce, identity, and +certificate-state transitions. For an already-authenticated row, the default +manager coalesces only timestamp-only usage touches for up to one minute. This +avoids a synchronous replicated write on every RPC while keeping durable expiry +within a bounded minute of the most recent use. Use `touchIntervalMs: 0` to +retain exact per-request timestamp persistence. + Both stages return HTTP 429 with `ERR_RATE_LIMITED`. For multi-process or multi-replica deployments, configure a shared `express-rate-limit` store in both options so limits are aggregate rather than per process. Never use a diff --git a/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts b/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts new file mode 100644 index 000000000..97d944c48 --- /dev/null +++ b/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts @@ -0,0 +1,607 @@ +import { Beef, MerklePath, Script, Telemetry, type TelemetryEvent, Transaction, Utils, Validation } from '@bsv/sdk' +import { once } from 'node:events' +import { performance } from 'node:perf_hooks' +import { _tu, TestWalletNoSetup, TestWalletOnly } from '../test/utils/TestUtilsWalletStorage' +import { KnexSessionManager } from '../src/storage/remoting/KnexSessionManager' +import { StorageServer, WalletStorageServerOptions } from '../src/storage/remoting/StorageServer' +import { StorageKnex } from '../src/storage/StorageKnex' +import { managedChangeOutputFields } from '../src/storage/methods/managedChange' +import { + TableOutput, + TableOutputBasket, + TableProvenTx, + TableTransaction +} from '../src/storage/schema/tables' +import { ScriptTemplateBRC29 } from '../src/utility/ScriptTemplateBRC29' +import { BdkVerifier } from '@bsv/verifast' + +interface Measurement { + candidateCount: number + selectedInputCount: number + distinctSourceCount: number + elapsedMs: number + queryCount: number + databaseTransactions: number + databaseMs: number + resultBeefBytes: number + phaseMs?: Record +} + +interface MeasurementSummary { + samples: number + p50Ms: number + p95Ms: number + maxMs: number + p50DatabaseMs: number + p95DatabaseMs: number + maxQueryCount: number + maxDatabaseTransactions: number + phaseP50Ms: Record + phaseP95Ms: Record +} + +interface BenchmarkContext { + ctx: TestWalletNoSetup + basket: TableOutputBasket + storageEvents: TelemetryEvent[] +} + +interface QueryProbe { + stop: () => { queryCount: number, databaseTransactions: number, databaseMs: number } +} + +function makeSourceTransaction ( + index: number, + satoshis: number, + lockingScript: Script +): Transaction { + const transaction = new Transaction() + transaction.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromHex(`04${index.toString(16).padStart(8, '0')}`), + sequence: 0xffffffff + }) + transaction.addOutput({ satoshis, lockingScript }) + return transaction +} + +function makeBenchmarkMerklePaths (txids: string[], height: number, seed: number): MerklePath[] { + const sharedLevels = Math.log2(txids.length) + if (!Number.isInteger(sharedLevels)) throw new Error('benchmark proof group must be a power of two') + const path = Array.from({ length: 24 }, (_, level) => { + const siblingHash = (BigInt(seed + 1) * 10_000n + BigInt(level + 1)) + .toString(16) + .padStart(64, '0') + .slice(-64) + if (level === 0) return txids.map((hash, offset) => ({ offset, hash, txid: true })) + if (level < sharedLevels) return [] + return [{ offset: 1, hash: siblingHash }] + }) + const compound = new MerklePath(height, path) + return txids.map(txid => compound.extract([txid])) +} + +async function createBenchmarkContext (): Promise { + const databaseName = process.env.WALLET_TOOLBOX_BENCH_MYSQL_DATABASE ?? 'createActionBeefBench' + const ctx = process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' + ? await _tu.createLegacyWalletMySQLCopy(databaseName, 'legacy') + : await _tu.createLegacyWalletSQLiteCopy(databaseName, 'legacy') + const storageEvents: TelemetryEvent[] = [] + Reflect.set(ctx.activeStorage, 'telemetry', new Telemetry({ + sink: { capture: event => storageEvents.push({ ...event }) } + })) + ctx.activeStorage.feeModel = { model: 'sat/kb', value: 100 } + const basket = (await ctx.activeStorage.findOutputBaskets({ + partial: { userId: ctx.userId, name: 'default' } + }))[0] as TableOutputBasket + await ctx.activeStorage.updateOutputBasket(basket.basketId, { + numberOfDesiredUTXOs: 0, + minimumDesiredUTXOValue: 1 + }) + return { ctx, basket, storageEvents } +} + +function collectPhaseDurations (events: TelemetryEvent[], firstEvent: number): Record { + const phaseMs: Record = {} + for (const event of events.slice(firstEvent)) { + if (event.type !== 'span' || event.durationMs == null) continue + const rpcMethod = event.attributes?.['rpc.method'] + const name = typeof rpcMethod === 'string' ? `${event.name}.${rpcMethod}` : event.name + phaseMs[name] = (phaseMs[name] ?? 0) + event.durationMs + } + return phaseMs +} + +async function replaceFundingCandidatesWithDistinctProvenSources ( + setup: BenchmarkContext, + candidateCount: number, + outputSatoshis: number, + generation: number, + signableManagedOutputs = false +): Promise { + const { ctx, basket } = setup + const sourceTxids: string[] = [] + const changeKeys = ctx.wallet.getClientChangeKeyPair() + const derivationPrefix = `beef-benchmark-prefix-${generation}` + const prepared = Array.from({ length: candidateCount }, (_, index) => { + const uniqueIndex = generation * candidateCount + index + const derivationSuffix = `beef-benchmark-${uniqueIndex}` + const lockingScript = signableManagedOutputs + ? new ScriptTemplateBRC29({ + derivationPrefix, + derivationSuffix, + keyDeriver: ctx.keyDeriver + }).lock(changeKeys.privateKey, changeKeys.publicKey) + : Script.fromHex('51') + const source = makeSourceTransaction(uniqueIndex, outputSatoshis, lockingScript) + return { + uniqueIndex, + derivationSuffix, + lockingScript, + source, + rawTx: source.toBinary(), + txid: source.id('hex'), + merklePath: undefined as MerklePath | undefined + } + }) + const proofGroupSize = 8 + for (let offset = 0; offset < prepared.length; offset += proofGroupSize) { + const group = prepared.slice(offset, offset + proofGroupSize) + const paths = makeBenchmarkMerklePaths( + group.map(source => source.txid), + 800_000 + generation * Math.ceil(candidateCount / proofGroupSize) + offset / proofGroupSize, + generation * candidateCount + offset + ) + for (let index = 0; index < group.length; index++) group[index].merklePath = paths[index] + } + + // Benchmark fixture setup is deliberately one transaction. In particular, + // do not spend hundreds of PXC commits preparing an unmeasured sample. + await ctx.activeStorage.transaction(async trx => { + const existing = await ctx.activeStorage.findOutputs({ + partial: { userId: ctx.userId, basketId: basket.basketId, spendable: true }, + noScript: true, + trx + }) + for (const output of existing) { + await ctx.activeStorage.updateOutput(output.outputId, { spendable: false }, trx) + } + + for (const preparedSource of prepared) { + const { derivationSuffix, lockingScript, source, rawTx, txid } = preparedSource + const merklePath = preparedSource.merklePath! + const now = new Date() + const proven: TableProvenTx = { + created_at: now, + updated_at: now, + provenTxId: 0, + txid, + height: merklePath.blockHeight, + index: 0, + merklePath: merklePath.toBinary(), + rawTx, + blockHash: txid, + merkleRoot: merklePath.computeRoot(txid) + } + proven.provenTxId = await ctx.activeStorage.insertProvenTx(proven, trx) + const transaction: TableTransaction = { + created_at: now, + updated_at: now, + transactionId: 0, + userId: ctx.userId, + provenTxId: proven.provenTxId, + status: 'completed', + reference: Utils.toBase64(Utils.toArray(txid, 'hex').slice(0, 12)), + isOutgoing: true, + satoshis: outputSatoshis, + description: 'proof-bearing funding benchmark source', + version: source.version, + lockTime: source.lockTime, + txid, + rawTx + } + transaction.transactionId = await ctx.activeStorage.insertTransaction(transaction, trx) + const output: TableOutput = { + outputId: 0, + userId: ctx.userId, + transactionId: transaction.transactionId, + basketId: basket.basketId, + spendable: true, + spentBy: undefined, + satoshis: outputSatoshis, + vout: 0, + txid, + lockingScript: lockingScript.toBinary(), + scriptLength: lockingScript.length, + derivationPrefix, + derivationSuffix, + outputDescription: 'proof-bearing funding candidate', + ...managedChangeOutputFields, + created_at: now, + updated_at: now + } + await ctx.activeStorage.insertOutput(output, trx) + sourceTxids.push(txid) + } + }) + return sourceTxids +} + +function createActionArgs (satoshis = 150_000) { + return Validation.validateCreateActionArgs({ + outputs: [{ + satoshis, + lockingScript: '51', + outputDescription: 'proof-bearing benchmark output' + }], + description: 'createAction proof-bearing fragmented funding benchmark', + options: { noSend: true, randomizeOutputs: false, returnTXIDOnly: false } + }) +} + +function startQueryProbe (setup: BenchmarkContext): QueryProbe { + let queryCount = 0 + let databaseTransactions = 0 + let databaseMs = 0 + const started = new Map() + const countQuery = (query: { sql?: string, __knexQueryUid?: string }): void => { + queryCount++ + if (/^begin\b/i.test(query.sql?.trim() ?? '')) databaseTransactions++ + if (query.__knexQueryUid != null) { + const sql = query.sql?.trim().toLowerCase() ?? '' + const table = /(?:from|into|update)\s+[`"]?([a-z_]+)/.exec(sql)?.[1] ?? 'transaction' + const operation = `${sql.split(/\s+/, 1)[0] || 'unknown'} ${table}` + started.set(query.__knexQueryUid, { at: performance.now(), operation }) + } + if (process.env.WALLET_TOOLBOX_BENCH_QUERIES === 'true') { + process.stdout.write(`${String(query.sql).replace(/\s+/g, ' ').trim()}\n`) + } + } + const finishQuery = (_response: unknown, query: { __knexQueryUid?: string }): void => { + const uid = query.__knexQueryUid + if (uid == null) return + const start = started.get(uid) + if (start != null) { + const elapsed = performance.now() - start.at + databaseMs += elapsed + if (process.env.WALLET_TOOLBOX_BENCH_QUERIES === 'true') { + process.stdout.write(`${start.operation}: ${elapsed.toFixed(3)} ms\n`) + } + } + started.delete(uid) + } + setup.ctx.activeStorage.knex.on('query', countQuery) + setup.ctx.activeStorage.knex.on('query-response', finishQuery) + return { + stop: () => { + setup.ctx.activeStorage.knex.off('query', countQuery) + setup.ctx.activeStorage.knex.off('query-response', finishQuery) + return { queryCount, databaseTransactions, databaseMs } + } + } +} + +async function measureStorageCreateAction ( + setup: BenchmarkContext, + candidateCount: number +): Promise { + const firstEvent = setup.storageEvents.length + const probe = startQueryProbe(setup) + const start = performance.now() + const result = await setup.ctx.activeStorage.createAction( + { userId: setup.ctx.userId }, + createActionArgs() + ) + const elapsedMs = performance.now() - start + const query = probe.stop() + return { + candidateCount, + selectedInputCount: result.inputs.length, + distinctSourceCount: new Set(result.inputs.map(input => input.sourceTxid)).size, + elapsedMs, + ...query, + resultBeefBytes: result.inputBeef?.length ?? 0, + phaseMs: collectPhaseDurations(setup.storageEvents, firstEvent) + } +} + +async function createRemoteClient ( + setup: BenchmarkContext +): Promise<{ + client: TestWalletOnly + server: StorageServer + verifier: BdkVerifier + verifyDigestBatch: jest.SpiedFunction + events: TelemetryEvent[] + serverEvents: TelemetryEvent[] +}> { + const events: TelemetryEvent[] = [] + const serverEvents: TelemetryEvent[] = [] + const telemetry = { sink: { capture: (event: Readonly) => events.push({ ...event }) } } + const serverTelemetry = { + sink: { capture: (event: Readonly) => serverEvents.push({ ...event }) } + } + // Jest VM modules create a separate typed-array realm for worker replies; + // use the same packed WASM batch lane without worker fan-out here. + const verifier = new BdkVerifier({ batchWorkers: 1 }) + await verifier.preload() + const verifyDigestBatch = jest.spyOn(verifier, 'verifyDigestBatch') + const options: WalletStorageServerOptions = { + port: 0, + wallet: setup.ctx.wallet, + monetize: false, + logRpcRequests: false, + sessionManager: new KnexSessionManager(setup.ctx.activeStorage.knex), + adminIdentityKeys: [], + telemetry: serverTelemetry, + calculateRequestPrice: async () => 0 + } + const server = new StorageServer(setup.ctx.activeStorage, options) + server.start() + if (!server.server.listening) await once(server.server, 'listening') + const address = server.server.address() + if (address == null || typeof address === 'string') throw new Error('benchmark storage server did not bind') + const client = await _tu.createTestWalletWithStorageClient({ + rootKeyHex: setup.ctx.rootKey.toHex(), + endpointUrl: `http://localhost:${address.port}`, + chain: setup.ctx.chain, + actionBatchMode: 'legacy', + telemetry, + scriptVerifier: verifier + }) + await client.storage.getAuth(true) + events.length = 0 + serverEvents.length = 0 + return { client, server, verifier, verifyDigestBatch, events, serverEvents } +} + +async function measureRemoteWalletCreateAction ( + setup: BenchmarkContext, + client: TestWalletOnly, + events: TelemetryEvent[], + serverEvents: TelemetryEvent[], + candidateCount: number +): Promise { + const firstEvent = events.length + const firstServerEvent = serverEvents.length + const firstStorageEvent = setup.storageEvents.length + const probe = startQueryProbe(setup) + const start = performance.now() + const result = await client.wallet.createAction({ + outputs: [{ + satoshis: 150_000, + lockingScript: '51', + outputDescription: 'proof-bearing benchmark output' + }], + description: 'authenticated remote proof-bearing fragmented funding benchmark', + options: { noSend: true, randomizeOutputs: false, returnTXIDOnly: false } + }) + const elapsedMs = performance.now() - start + const query = probe.stop() + if (result.tx == null || result.txid == null) throw new Error('remote benchmark did not return an atomic BEEF') + const resultBeef = Beef.fromBinary(result.tx) + const transaction = resultBeef.findTxid(result.txid)?.tx + if (transaction == null) throw new Error('remote benchmark result transaction is missing') + const phaseMs = collectPhaseDurations(events, firstEvent) + const serverPhaseMs = collectPhaseDurations(serverEvents, firstServerEvent) + for (const [name, durationMs] of Object.entries(serverPhaseMs)) { + phaseMs[`server.${name}`] = durationMs + } + const storagePhaseMs = collectPhaseDurations(setup.storageEvents, firstStorageEvent) + for (const [name, durationMs] of Object.entries(storagePhaseMs)) { + phaseMs[`server.storage.${name}`] = durationMs + } + return { + candidateCount, + selectedInputCount: transaction.inputs.length, + distinctSourceCount: new Set(transaction.inputs.map(input => input.sourceTXID)).size, + elapsedMs, + ...query, + resultBeefBytes: result.tx?.length ?? 0, + phaseMs + } +} + +function percentile (values: number[], percentileValue: number): number { + const sorted = [...values].sort((a, b) => a - b) + return sorted[Math.max(0, Math.ceil(percentileValue * sorted.length) - 1)] +} + +function summarize (measurements: Measurement[]): MeasurementSummary { + const elapsed = measurements.map(measurement => measurement.elapsedMs) + const database = measurements.map(measurement => measurement.databaseMs) + const phaseNames = new Set(measurements.flatMap(measurement => Object.keys(measurement.phaseMs ?? {}))) + const phaseP50Ms: Record = {} + const phaseP95Ms: Record = {} + for (const phase of phaseNames) { + const values = measurements.map(measurement => measurement.phaseMs?.[phase] ?? 0) + phaseP50Ms[phase] = percentile(values, 0.5) + phaseP95Ms[phase] = percentile(values, 0.95) + } + return { + samples: measurements.length, + p50Ms: percentile(elapsed, 0.5), + p95Ms: percentile(elapsed, 0.95), + maxMs: Math.max(...elapsed), + p50DatabaseMs: percentile(database, 0.5), + p95DatabaseMs: percentile(database, 0.95), + maxQueryCount: Math.max(...measurements.map(measurement => measurement.queryCount)), + maxDatabaseTransactions: Math.max(...measurements.map(measurement => measurement.databaseTransactions)), + phaseP50Ms, + phaseP95Ms + } +} + +describe('createAction proof-bearing fragmented funding benchmark', () => { + jest.setTimeout(600_000) + + test('builds deterministic same-root proof groups for benchmark fixtures', () => { + const txids = ['01', '02', '03', '04'].map(prefix => prefix.padEnd(64, '0')) + const paths = makeBenchmarkMerklePaths(txids, 900_000, 7) + expect(paths).toHaveLength(txids.length) + expect(new Set(paths.map((path, index) => path.computeRoot(txids[index]))).size).toBe(1) + }) + + const localTest = process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' ? test.skip : test + localTest('records complete createAction latency for many distinct proven sources', async () => { + const setup = await createBenchmarkContext() + try { + await replaceFundingCandidatesWithDistinctProvenSources(setup, 178, 1_000, 0) + const measurement = await measureStorageCreateAction(setup, 178) + expect(measurement.selectedInputCount).toBeGreaterThan(100) + expect(measurement.distinctSourceCount).toBe(measurement.selectedInputCount) + expect(measurement.resultBeefBytes).toBeGreaterThan(0) + expect(measurement.queryCount).toBeLessThanOrEqual(15) + process.stdout.write(`${JSON.stringify({ measurement }, null, 2)}\n`) + } finally { + await setup.ctx.wallet.destroy() + } + }) + + const existingDatabase = process.env.WALLET_TOOLBOX_BENCH_EXISTING_DATABASE === 'true' + const pxcTest = process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' && !existingDatabase ? test : test.skip + pxcTest('holds the direct and authenticated remote p50/p95 latency gates on PXC', async () => { + const setup = await createBenchmarkContext() + const samples = Number(process.env.WALLET_TOOLBOX_BENCH_SAMPLES ?? 20) + let remote: Awaited> | undefined + try { + const directMeasurements: Measurement[] = [] + for (let sample = 0; sample < samples; sample++) { + await replaceFundingCandidatesWithDistinctProvenSources(setup, 178, 1_000, sample + 1) + await new Promise(resolve => setTimeout(resolve, 1_250)) + directMeasurements.push(await measureStorageCreateAction(setup, 178)) + } + + remote = await createRemoteClient(setup) + const remoteMeasurements: Measurement[] = [] + for (let sample = 0; sample < samples; sample++) { + await replaceFundingCandidatesWithDistinctProvenSources(setup, 178, 1_000, sample + 1_000, true) + await new Promise(resolve => setTimeout(resolve, 1_250)) + remoteMeasurements.push(await measureRemoteWalletCreateAction( + setup, + remote.client, + remote.events, + remote.serverEvents, + 178 + )) + } + + const typicalRemoteMeasurements: Measurement[] = [] + for (let sample = 0; sample < samples; sample++) { + await replaceFundingCandidatesWithDistinctProvenSources(setup, 8, 200_000, sample + 2_000, true) + await new Promise(resolve => setTimeout(resolve, 1_250)) + typicalRemoteMeasurements.push(await measureRemoteWalletCreateAction( + setup, + remote.client, + remote.events, + remote.serverEvents, + 8 + )) + } + + const direct = summarize(directMeasurements) + const authenticatedRemote = summarize(remoteMeasurements) + const typicalAuthenticatedRemote = summarize(typicalRemoteMeasurements) + const digestVerdicts = (await Promise.all( + remote.verifyDigestBatch.mock.results.map(async result => await result.value) + )).flat() + process.stdout.write(`${JSON.stringify({ + direct, + authenticatedRemote, + typicalAuthenticatedRemote, + digestVerificationBatches: remote.verifyDigestBatch.mock.calls.length, + digestVerificationCount: digestVerdicts.length, + digestVerificationFailures: digestVerdicts.filter(valid => !valid).length + }, null, 2)}\n`) + expect(direct.p50Ms).toBeLessThan(150) + expect(direct.p95Ms).toBeLessThan(500) + expect(authenticatedRemote.p50Ms).toBeLessThan(450) + expect(authenticatedRemote.p95Ms).toBeLessThan(500) + expect(authenticatedRemote.maxQueryCount).toBeLessThanOrEqual(33) + expect(typicalAuthenticatedRemote.p50Ms).toBeLessThan(100) + expect(typicalAuthenticatedRemote.p95Ms).toBeLessThan(150) + expect(remoteMeasurements.every(measurement => measurement.selectedInputCount === 153)).toBe(true) + expect(remoteMeasurements.every(measurement => measurement.distinctSourceCount === 153)).toBe(true) + expect(typicalRemoteMeasurements.every(measurement => measurement.selectedInputCount === 1)).toBe(true) + expect(typicalRemoteMeasurements.every(measurement => measurement.distinctSourceCount === 1)).toBe(true) + expect(remote.verifyDigestBatch).toHaveBeenCalledTimes(samples * 2) + expect(digestVerdicts).toHaveLength(samples * 154) + expect(digestVerdicts).not.toContain(false) + } finally { + if (remote != null) { + await remote.client.wallet.destroy() + await remote.server.close() + remote.verifier.dispose() + } + await setup.ctx.wallet.destroy() + } + }) + + const existingDatabaseTest = existingDatabase ? test : test.skip + existingDatabaseTest('measures against an isolated production-shaped database copy', async () => { + const databaseName = process.env.WALLET_TOOLBOX_BENCH_MYSQL_DATABASE + if (databaseName == null || databaseName.length === 0) throw new Error('benchmark database name is required') + const events: TelemetryEvent[] = [] + const storage = new StorageKnex({ + ...StorageKnex.defaultOptions(), + knex: _tu.createLocalMySQL(databaseName), + chain: 'main', + telemetry: { sink: { capture: event => events.push({ ...event }) } } + }) + try { + await storage.makeAvailable() + storage.feeModel = { model: 'sat/kb', value: 100 } + const user = await storage.knex('users').select('userId').first() + const basket = await storage.knex('output_baskets').select('basketId').where({ name: 'default' }).first() + if (user?.userId == null || basket?.basketId == null) throw new Error('production-shaped fixture is incomplete') + const samples = Number(process.env.WALLET_TOOLBOX_BENCH_EXISTING_SAMPLES ?? 5) + const measurements: Measurement[] = [] + for (let sample = 0; sample < samples; sample++) { + const available = await storage.findAvailableManagedChangeInputCandidates(user.userId, basket.basketId) + // Keep enough headroom for a large-input transaction fee while still + // forcing the fragmented cohort to contribute well over 100 inputs. + const target = Math.max(1, Math.floor(available.reduce((sum, output) => sum + output.satoshis, 0) * 0.7)) + const setup = { ctx: { activeStorage: storage, userId: user.userId } } as unknown as BenchmarkContext + events.length = 0 + const probe = startQueryProbe(setup) + const start = performance.now() + const result = await storage.createAction({ userId: user.userId }, createActionArgs(target)) + const elapsedMs = performance.now() - start + const query = probe.stop() + const phaseMs: Record = {} + for (const event of events) { + if (event.type !== 'span' || event.durationMs == null) continue + phaseMs[event.name] = (phaseMs[event.name] ?? 0) + event.durationMs + } + measurements.push({ + candidateCount: available.length, + selectedInputCount: result.inputs.length, + distinctSourceCount: new Set(result.inputs.map(input => input.sourceTxid)).size, + elapsedMs, + ...query, + resultBeefBytes: result.inputBeef?.length ?? 0, + phaseMs + }) + await expect(storage.abortAction( + { userId: user.userId }, + { reference: result.reference } + )).resolves.toEqual({ aborted: true }) + } + const summary = summarize(measurements) + process.stdout.write(`${JSON.stringify({ + productionShaped: summary, + minSelectedInputCount: Math.min(...measurements.map(measurement => measurement.selectedInputCount)), + minDistinctSourceCount: Math.min(...measurements.map(measurement => measurement.distinctSourceCount)) + }, null, 2)}\n`) + expect(measurements.every(measurement => measurement.candidateCount > 100)).toBe(true) + expect(measurements.every(measurement => measurement.selectedInputCount > 100)).toBe(true) + expect(measurements.every(measurement => measurement.resultBeefBytes > 0)).toBe(true) + expect(summary.maxQueryCount).toBeLessThanOrEqual(11) + expect(summary.p50Ms).toBeLessThan(100) + expect(summary.p95Ms).toBeLessThan(500) + } finally { + await storage.destroy() + } + }) +}) diff --git a/packages/wallet/wallet-toolbox/client/package.json b/packages/wallet/wallet-toolbox/client/package.json index d9eed7c21..d33ff4c6c 100644 --- a/packages/wallet/wallet-toolbox/client/package.json +++ b/packages/wallet/wallet-toolbox/client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-client", - "version": "2.4.22", + "version": "2.5.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 753536adb..4e499311b 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1502000, + "raw": 1525000, "gzip": 360000, - "brotli": 280000 + "brotli": 283000 }, "esbuild": { - "raw": 1175000, + "raw": 1192000, "gzip": 330000, "brotli": 265000 } diff --git a/packages/wallet/wallet-toolbox/mobile/package.json b/packages/wallet/wallet-toolbox/mobile/package.json index c130a0e83..af43a3b0b 100644 --- a/packages/wallet/wallet-toolbox/mobile/package.json +++ b/packages/wallet/wallet-toolbox/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-mobile", - "version": "2.4.22", + "version": "2.5.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index a095b9b53..11cba5bd1 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox", - "version": "2.4.22", + "version": "2.5.0", "sideEffects": false, "type": "commonjs", "engines": { @@ -60,6 +60,7 @@ "operator": "pnpm operator:build && node .operator-build/cli.js", "bench:action-batch": "pnpm build && jest --runInBand --runTestsByPath benchmarks/action-batch.bench.test.ts --testPathIgnorePatterns=man.test.ts", "bench:create-action-funding": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-funding.bench.test.ts --testPathIgnorePatterns=man.test.ts", + "bench:create-action-beef": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-beef.bench.test.ts --testPathIgnorePatterns=man.test.ts", "format:check": "pnpm --workspace-root exec prettier --check \"packages/wallet/wallet-toolbox/{README.md,jest.config.cjs,package.json,tsconfig*.json}\"", "lint": "oxlint src test benchmarks examples operator --deny-warnings", "lint:ci": "pnpm lint", @@ -91,6 +92,7 @@ }, "devDependencies": { "@bsv/sdk": "workspace:^", + "@bsv/verifast": "workspace:^", "@jest/globals": "^30.4.1", "@types/body-parser": "^1.19.6", "@types/cors": "^2.8.19", diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index 4e2b753f4..788e4dda3 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -109,6 +109,27 @@ import { getResultBeef } from './signer/methods/resultBeef' import type { ValidListOutputsArgs } from '@bsv/sdk/wallet/validationHelpers' import { ActionBatchController, ActionBatchMode } from './signer/actionBatch/ActionBatchWorkspace' +function prepareKnownTxidsForCreateAction(wallet: Wallet, args: CreateActionArgs): void { + if (!wallet.autoKnownTxids || args.options?.knownTxids != null) return + if (!wallet.telemetry.enabled) { + args.options!.knownTxids = wallet.getKnownTxids(args.options?.knownTxids) + return + } + args.options!.knownTxids = wallet.telemetry.withSpan( + 'wallet.create_action.prepare_known_txids', + { + component: 'wallet-toolbox', + carrier: args, + attributes: { 'beef.tx_count': wallet.beef.txs.length } + }, + span => { + const knownTxids = wallet.getKnownTxids(args.options?.knownTxids) + span.end({ attributes: { 'beef.known_txid_count': knownTxids.length } }) + return knownTxids + } + ) +} + /** * The preferred means of constructing a `Wallet` is with a `WalletArgs` instance. */ @@ -204,9 +225,9 @@ export interface WalletArgs { */ makeLogger?: MakeWalletLogger /** - * Internal Wallet Toolbox optimization policy. `auto` negotiates the optional - * action-batch storage capability; `legacy` always uses per-action storage. - * This does not change the BRC-100 wallet interface. + * Internal Wallet Toolbox optimization policy. `auto` (the default) + * negotiates the optional action-batch storage capability; `legacy` always + * uses per-action storage. This does not change the BRC-100 wallet interface. */ actionBatchMode?: ActionBatchMode /** @@ -915,6 +936,8 @@ export class Wallet implements WalletInterface, ProtoWallet { if (this.returnTxidOnly) return beef const b = parsedBeef ?? Beef.fromBinary(beef) if (!b.atomicTxid) throw new WERR_INTERNAL() + const hasUnknownTxidOnly = b.txs.some(btx => btx.isTxidOnly && !knownTxids?.includes(btx.txid)) + if (!hasUnknownTxidOnly) return beef return this.verifyReturnedTxidOnly(b, knownTxids).toBinaryAtomic(b.atomicTxid) } @@ -960,25 +983,7 @@ export class Wallet implements WalletInterface, ProtoWallet { args.options ??= {} args.options.trustSelf ||= this.trustSelf - if (this.autoKnownTxids && args.options.knownTxids == null) { - if (this.telemetry.enabled) { - args.options.knownTxids = this.telemetry.withSpan( - 'wallet.create_action.prepare_known_txids', - { - component: 'wallet-toolbox', - carrier: args, - attributes: { 'beef.tx_count': this.beef.txs.length } - }, - span => { - const knownTxids = this.getKnownTxids(args.options?.knownTxids) - span.end({ attributes: { 'beef.known_txid_count': knownTxids.length } }) - return knownTxids - } - ) - } else { - args.options.knownTxids = this.getKnownTxids(args.options.knownTxids) - } - } + prepareKnownTxidsForCreateAction(this, args) const { auth, vargs } = this.validateAuthAndArgs(args, Validation.validateCreateActionArgs, logger) logger?.log('validated args') @@ -992,10 +997,40 @@ export class Wallet implements WalletInterface, ProtoWallet { logger?.log('action created') const resultBeef = getResultBeef(r) - if (r.tx != null) this.beef.mergeBeefFromParty(this.storageParty, resultBeef ?? r.tx) + if (r.tx != null) { + const merge = (): void => this.beef.mergeBeefFromParty(this.storageParty, resultBeef ?? r.tx!) + if (this.telemetry.enabled) { + this.telemetry.withSpan( + 'wallet.create_action.merge_result_beef', + { + component: 'wallet-toolbox', + carrier: args, + attributes: { + 'beef.retained_tx_count_before': this.beef.txs.length, + 'beef.result_byte_count': r.tx.length + } + }, + merge + ) + } else { + merge() + } + } if (r.tx != null) { - r.tx = this.verifyReturnedTxidOnlyAtomicBEEF(r.tx, args.options?.knownTxids, resultBeef) + const verify = (): AtomicBEEF => + this.verifyReturnedTxidOnlyAtomicBEEF(r.tx!, args.options?.knownTxids, resultBeef) + r.tx = this.telemetry.enabled + ? this.telemetry.withSpan( + 'wallet.create_action.verify_result_beef', + { + component: 'wallet-toolbox', + carrier: args, + attributes: { 'beef.result_byte_count': r.tx.length } + }, + verify + ) + : verify() logger?.log('verify returned AtomicBEEF') } diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts index a341018da..e1f857249 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts @@ -453,6 +453,7 @@ export interface FindProvenTxReqsArgs extends FindSincePagedArgs { } export interface FindProvenTxsArgs extends FindSincePagedArgs { partial: Partial + txids?: string[] } export interface FindStaleMerkleRootsArgs { height: number diff --git a/packages/wallet/wallet-toolbox/src/signer/methods/__tests/completeSignedTransaction.verify.test.ts b/packages/wallet/wallet-toolbox/src/signer/methods/__tests/completeSignedTransaction.verify.test.ts index 70fa4257e..7e8efe0c3 100644 --- a/packages/wallet/wallet-toolbox/src/signer/methods/__tests/completeSignedTransaction.verify.test.ts +++ b/packages/wallet/wallet-toolbox/src/signer/methods/__tests/completeSignedTransaction.verify.test.ts @@ -1,7 +1,14 @@ import { Beef, + BigNumber, + ECDSA, + P2PKH, + PrivateKey, + PublicKey, Script, + ScriptEvaluationError, ScriptResourceLimitError, + Signature, Transaction } from '@bsv/sdk' import { @@ -51,6 +58,70 @@ function verificationFixture ( return { beef, txid: tx.id('hex') } } +function scriptShapeFixture (lockingScript: Script, unlockingScript: Script): { + beef: Beef + txid: string +} { + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 2, lockingScript }) + const tx = new Transaction() + tx.addInput({ sourceTransaction: source, sourceOutputIndex: 0, unlockingScript }) + tx.addOutput({ satoshis: 1, lockingScript: Script.fromASM('OP_TRUE') }) + const beef = new Beef() + beef.mergeTransaction(tx) + return { beef, txid: tx.id('hex') } +} + +function structurallyMutableBeef (source: Beef, txid: string): { beef: Beef, tx: Transaction } { + const tx = source.findTxid(txid)?.tx + if (tx == null) throw new Error('test transaction missing from BEEF') + const txs = source.txs.map(item => ({ txid: item.txid, tx: item.tx })) + return { + beef: { + findTxid: (candidate: string) => txs.find(item => item.txid === candidate), + txs + } as unknown as Beef, + tx + } +} + +async function p2pkhVerificationFixture(resolvedInputs = 2): Promise<{ + beef: Beef + txid: string +}> { + const key = new PrivateKey(42) + const p2pkh = new P2PKH() + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromASM('OP_TRUE') + }) + for (let index = 0; index < resolvedInputs; index++) { + source.addOutput({ satoshis: 2, lockingScript: p2pkh.lock(key.toAddress()) }) + } + + const tx = new Transaction() + for (let index = 0; index < resolvedInputs; index++) { + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: index, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + } + tx.addOutput({ satoshis: 1, lockingScript: Script.fromASM('OP_TRUE') }) + await tx.sign() + + const beef = new Beef() + beef.mergeTransaction(tx) + return { beef, txid: tx.id('hex') } +} + describe('verifyUnlockScripts', () => { it('verifies every resolvable input in explicit consensus context', async () => { const { beef, txid } = verificationFixture(true) @@ -117,4 +188,307 @@ describe('verifyUnlockScripts', () => { ) expect(verifier.verifySpend).not.toHaveBeenCalled() }) + + it('verifies a fully resolved transaction once when the backend supports it', async () => { + const { beef, txid } = verificationFixture(false, 2) + const verifyScripts = jest.fn(async () => true) + const verifySpendsBatch = jest.fn(async () => [true, true]) + const verifier = { + shouldVerifyScripts: jest.fn(() => true), + verifyScripts, + shouldVerifySpend: jest.fn(() => true), + verifySpend: jest.fn(async () => true), + verifySpendsBatch + } + + await expect(verifyUnlockScripts(txid, beef, verifier)).resolves.toEqual({ + verifiedInputs: 2, + skippedInputs: 0 + }) + expect(verifyScripts).toHaveBeenCalledTimes(1) + expect(verifyScripts).toHaveBeenCalledWith(expect.objectContaining({ + tx: beef.findTxid(txid)?.tx, + blockHeight: 943816, + consensus: true + })) + expect(verifySpendsBatch).not.toHaveBeenCalled() + }) + + it('retains per-input diagnostics after a false whole-transaction verdict', async () => { + const { beef, txid } = verificationFixture(false, 2) + const verifySpendsBatch = jest.fn(async () => [true, false]) + const verifier = { + verifyScripts: jest.fn(async () => false), + shouldVerifySpend: jest.fn(() => true), + verifySpend: jest.fn(async () => true), + verifySpendsBatch + } + + await expect(verifyUnlockScripts(txid, beef, verifier)).rejects.toThrow('inputs[1].unlockScript') + expect(verifySpendsBatch).toHaveBeenCalledTimes(1) + }) + + it('retains per-input diagnostics after a whole-transaction script error', async () => { + const { beef, txid } = verificationFixture(false, 2) + const verifySpendsBatch = jest.fn(async () => [true, false]) + const scriptError = new ScriptEvaluationError({ + message: 'OP_VERIFY failed', + txid, + outputIndex: 0, + context: 'LockingScript', + programCounter: 0, + stackState: [], + altStackState: [], + ifStackState: [], + stackMem: 0, + altStackMem: 0 + }) + const verifier = { + verifyScripts: jest.fn(async () => { throw scriptError }), + shouldVerifySpend: jest.fn(() => true), + verifySpend: jest.fn(async () => true), + verifySpendsBatch + } + + await expect(verifyUnlockScripts(txid, beef, verifier)).rejects.toThrow('inputs[1].unlockScript') + expect(verifySpendsBatch).toHaveBeenCalledTimes(1) + }) + + it('retains known-source skipping instead of selecting whole-transaction verification', async () => { + const { beef, txid } = verificationFixture(true) + const verifyScripts = jest.fn(async () => true) + const verifier = { + verifyScripts, + shouldVerifySpend: jest.fn(() => true), + verifySpend: jest.fn(async () => true) + } + + await expect(verifyUnlockScripts(txid, beef, verifier)).resolves.toEqual({ + verifiedInputs: 1, + skippedInputs: 1 + }) + expect(verifyScripts).not.toHaveBeenCalled() + }) + + it('batch-verifies exact wallet P2PKH signatures without general script serialization', async () => { + const { beef, txid } = await p2pkhVerificationFixture(2) + const verifyDigestBatch = jest.fn(async items => items.map(item => ECDSA.verify( + new BigNumber(Array.from(item.digest)), + Signature.fromDER(Array.from(item.signature)), + PublicKey.fromDER(Array.from(item.publicKey)) + ))) + const verifyScripts = jest.fn(async () => true) + const verifySpendsBatch = jest.fn(async () => [true, true]) + const verifier = { + isReady: () => true, + supportsCrypto: (operation: string) => operation === 'verifyDigestBatch', + verifyDigestBatch, + verifyScripts, + shouldVerifySpend: jest.fn(() => true), + verifySpend: jest.fn(async () => true), + verifySpendsBatch + } + + await expect(verifyUnlockScripts(txid, beef, verifier)).resolves.toEqual({ + verifiedInputs: 2, + skippedInputs: 0 + }) + expect(verifyDigestBatch).toHaveBeenCalledTimes(1) + expect(verifyDigestBatch.mock.calls[0][0]).toHaveLength(2) + expect(verifyScripts).not.toHaveBeenCalled() + expect(verifySpendsBatch).not.toHaveBeenCalled() + }) + + it('falls through to per-input diagnostics after a false digest verdict', async () => { + const { beef, txid } = await p2pkhVerificationFixture(2) + const verifySpendsBatch = jest.fn(async () => [true, false]) + const verifier = { + isReady: () => true, + supportsCrypto: (operation: string) => operation === 'verifyDigestBatch', + verifyDigestBatch: jest.fn(async () => [true, false]), + shouldVerifySpend: jest.fn(() => true), + verifySpend: jest.fn(async () => true), + verifySpendsBatch + } + + await expect(verifyUnlockScripts(txid, beef, verifier)).rejects.toThrow('inputs[1].unlockScript') + expect(verifySpendsBatch).toHaveBeenCalledTimes(1) + }) + + it('rejects an invalid digest batch result count', async () => { + const { beef, txid } = await p2pkhVerificationFixture(2) + const verifier = { + isReady: () => true, + supportsCrypto: (operation: string) => operation === 'verifyDigestBatch', + verifyDigestBatch: jest.fn(async () => [true]), + verifySpend: jest.fn(async () => true) + } + + await expect(verifyUnlockScripts(txid, beef, verifier)) + .rejects.toThrow('invalid digest batch result count') + }) + + it.each([ + ['not ready', () => false, () => true], + ['unsupported', () => true, () => false] + ])('does not select a digest backend that is %s', async (_name, isReady, supportsCrypto) => { + const { beef, txid } = await p2pkhVerificationFixture(1) + const verifyDigestBatch = jest.fn(async () => [true]) + const verifySpendsBatch = jest.fn(async () => [true]) + + await expect(verifyUnlockScripts(txid, beef, { + isReady, + supportsCrypto, + verifyDigestBatch, + shouldVerifySpend: () => true, + verifySpend: async () => true, + verifySpendsBatch + })).resolves.toEqual({ verifiedInputs: 1, skippedInputs: 0 }) + expect(verifyDigestBatch).not.toHaveBeenCalled() + expect(verifySpendsBatch).toHaveBeenCalledTimes(1) + }) + + it('routes every non-canonical P2PKH shape through general script verification', async () => { + const canonical = await p2pkhVerificationFixture(1) + const canonicalTx = canonical.beef.findTxid(canonical.txid)?.tx + const canonicalInput = canonicalTx?.inputs[0] + const lock = Array.from(canonicalInput?.sourceTransaction?.outputs[0].lockingScript.toUint8Array() ?? []) + const unlock = Array.from(canonicalInput?.unlockingScript?.toUint8Array() ?? []) + expect(lock).toHaveLength(25) + expect(unlock.length).toBeGreaterThan(40) + + const lockVariants = [ + lock.slice(0, 24), + lock.map((byte, index) => index === 0 ? 0x75 : byte), + lock.map((byte, index) => index === 1 ? 0xa8 : byte), + lock.map((byte, index) => index === 2 ? 0x15 : byte), + lock.map((byte, index) => index === 23 ? 0x89 : byte), + lock.map((byte, index) => index === 24 ? 0xad : byte) + ] + const signatureLength = unlock[0] + const publicKeyOffset = 1 + signatureLength + 1 + const malformedSignature = [9, ...Array(9).fill(0), 33, ...unlock.slice(publicKeyOffset)] + const unlockVariants = [ + [], + [8, ...unlock.slice(1, 9), 33, ...unlock.slice(publicKeyOffset)], + [74, ...unlock.slice(1)], + unlock.slice(0, -1), + unlock.map((byte, index) => index === 1 + signatureLength ? 32 : byte), + unlock.map((byte, index) => index === publicKeyOffset ? 0x04 : byte), + unlock.map((byte, index) => index === unlock.length - 1 ? byte ^ 1 : byte), + malformedSignature, + unlock.map((byte, index) => index === signatureLength ? 0x42 : byte) + ] + const variants = [ + ...lockVariants.map(bytes => ({ lock: bytes, unlock })), + ...unlockVariants.map(bytes => ({ lock, unlock: bytes })) + ] + + for (const variant of variants) { + const { beef, txid } = scriptShapeFixture( + Script.fromBinary(variant.lock), + Script.fromBinary(variant.unlock) + ) + const verifyDigestBatch = jest.fn(async () => [true]) + const verifySpendsBatch = jest.fn(async () => [true]) + await expect(verifyUnlockScripts(txid, beef, { + isReady: () => true, + supportsCrypto: () => true, + verifyDigestBatch, + shouldVerifySpend: () => true, + verifySpend: async () => true, + verifySpendsBatch + })).resolves.toEqual({ verifiedInputs: 1, skippedInputs: 0 }) + expect(verifyDigestBatch).not.toHaveBeenCalled() + expect(verifySpendsBatch).toHaveBeenCalledTimes(1) + } + }) + + it('validates hydrated transaction structure before using accelerated backends', async () => { + const verifier = { + verifyScripts: jest.fn(async () => true), + verifySpend: jest.fn(async () => true) + } + await expect(verifyUnlockScripts('ff'.repeat(32), new Beef(), verifier)) + .rejects.toThrow('contained in beef') + + const missingSource = verificationFixture(false) + const mutableMissingSource = structurallyMutableBeef(missingSource.beef, missingSource.txid) + mutableMissingSource.tx.inputs[0].sourceTXID = undefined + await expect(verifyUnlockScripts(missingSource.txid, mutableMissingSource.beef, verifier)) + .rejects.toThrow('inputs[0].sourceTXID') + + const missingUnlock = verificationFixture(false) + const mutableMissingUnlock = structurallyMutableBeef(missingUnlock.beef, missingUnlock.txid) + mutableMissingUnlock.tx.inputs[0].unlockingScript = undefined + await expect(verifyUnlockScripts(missingUnlock.txid, mutableMissingUnlock.beef, verifier)) + .rejects.toThrow('inputs[0].unlockingScript') + + const badIndex = verificationFixture(false) + const mutableBadIndex = structurallyMutableBeef(badIndex.beef, badIndex.txid) + mutableBadIndex.tx.inputs[0].sourceOutputIndex = 99 + await expect(verifyUnlockScripts(badIndex.txid, mutableBadIndex.beef, verifier)) + .rejects.toThrow('reference an output') + }) + + it('supports whole-transaction batching and rejects malformed backend results', async () => { + const { beef, txid } = verificationFixture(false) + const verifyScriptsBatch = jest.fn(async () => [true, true]) + const verifier = { + verifyScripts: jest.fn(async () => true), + verifyScriptsBatch, + verifySpend: jest.fn(async () => true) + } + await expect(verifyUnlockScriptsBatch([txid, txid], beef, verifier)).resolves.toEqual([ + { verifiedInputs: 1, skippedInputs: 0 }, + { verifiedInputs: 1, skippedInputs: 0 } + ]) + expect(verifyScriptsBatch).toHaveBeenCalledTimes(1) + + verifyScriptsBatch.mockResolvedValueOnce([true]) + await expect(verifyUnlockScriptsBatch([txid, txid], beef, verifier)) + .rejects.toThrow('invalid transaction batch result count') + + const failure = new Error('whole backend unavailable') + verifyScriptsBatch.mockRejectedValueOnce(failure) + await expect(verifyUnlockScripts(txid, beef, verifier)).rejects.toBe(failure) + }) + + it('honors whole-transaction backend selection and empty batches', async () => { + const { beef, txid } = verificationFixture(false) + const verifyScripts = jest.fn(async () => true) + const verifySpendsBatch = jest.fn(async () => [true]) + await expect(verifyUnlockScripts(txid, beef, { + verifyScripts, + shouldVerifyScripts: () => false, + shouldVerifySpend: () => true, + verifySpend: async () => true, + verifySpendsBatch + })).resolves.toEqual({ verifiedInputs: 1, skippedInputs: 0 }) + expect(verifyScripts).not.toHaveBeenCalled() + expect(verifySpendsBatch).toHaveBeenCalledTimes(1) + + await expect(verifyUnlockScriptsBatch([], new Beef(), { + isReady: () => true, + supportsCrypto: () => true, + verifyDigestBatch: async () => [], + verifyScripts: async () => true, + verifySpend: async () => true + })).resolves.toEqual([]) + }) + + it('uses consensus defaults when canonical inputs omit optional numeric fields', async () => { + const fixture = await p2pkhVerificationFixture(1) + const mutable = structurallyMutableBeef(fixture.beef, fixture.txid) + const input = mutable.tx.inputs[0] + if (input.sourceTransaction == null) throw new Error('test source transaction missing') + input.sourceTransaction.outputs[input.sourceOutputIndex].satoshis = undefined + input.sequence = undefined + const verifyDigestBatch = jest.fn(async () => [true]) + const verifier = { verifySpend: async () => true, verifyDigestBatch } + + await expect(verifyUnlockScripts(fixture.txid, mutable.beef, verifier)) + .resolves.toEqual({ verifiedInputs: 1, skippedInputs: 0 }) + expect(verifyDigestBatch).toHaveBeenCalledTimes(1) + }) }) diff --git a/packages/wallet/wallet-toolbox/src/signer/methods/__tests__/signingTelemetry.test.ts b/packages/wallet/wallet-toolbox/src/signer/methods/__tests__/signingTelemetry.test.ts index a19ef57e9..cf9213e12 100644 --- a/packages/wallet/wallet-toolbox/src/signer/methods/__tests__/signingTelemetry.test.ts +++ b/packages/wallet/wallet-toolbox/src/signer/methods/__tests__/signingTelemetry.test.ts @@ -54,4 +54,48 @@ describe('signing telemetry', () => { await expect(completeSignedTransaction(prior, {}, wallet)).resolves.toBe(prior.tx) expect(sign).toHaveBeenCalledTimes(2) }) + + it('computes the invariant client change key pair once for every managed input', async () => { + const getClientChangeKeyPair = jest.fn(() => ({ + privateKey: '01'.padStart(64, '0'), + publicKey: '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + })) + const derivePrivateKey = jest.fn(() => ({ + toHex: () => '02'.padStart(64, '0') + })) + const inputs = [{}, {}, {}] + const prior = { + args: { inputs: [] }, + pdi: inputs.map((_, vin) => ({ + vin, + derivationPrefix: 'prefix', + derivationSuffix: `suffix-${vin}`, + sourceSatoshis: 1000, + lockingScript: '76a914000000000000000000000000000000000000000088ac' + })), + tx: { inputs, sign: jest.fn(async () => undefined) } + } as any + const events: TelemetryEvent[] = [] + const wallet = { + getClientChangeKeyPair, + keyDeriver: { + rootKey: { toHex: () => '01'.padStart(64, '0') }, + derivePrivateKey + }, + telemetry: new Telemetry({ sink: { capture: event => events.push(event) } }) + } as any + + await expect(completeSignedTransaction(prior, {}, wallet)).resolves.toBe(prior.tx) + + expect(getClientChangeKeyPair).toHaveBeenCalledTimes(1) + expect(derivePrivateKey).toHaveBeenCalledTimes(3) + expect(inputs.every(input => input.unlockingScriptTemplate != null)).toBe(true) + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ + name: 'wallet.crypto.prepare_unlocking_templates', + spanStatus: 'ok', + attributes: { 'crypto.managed_input_count': 3 } + }) + ])) + }) }) diff --git a/packages/wallet/wallet-toolbox/src/signer/methods/completeSignedTransaction.ts b/packages/wallet/wallet-toolbox/src/signer/methods/completeSignedTransaction.ts index 8ac8aaca1..b22608ac8 100644 --- a/packages/wallet/wallet-toolbox/src/signer/methods/completeSignedTransaction.ts +++ b/packages/wallet/wallet-toolbox/src/signer/methods/completeSignedTransaction.ts @@ -1,8 +1,8 @@ -import { SignActionSpend, Transaction } from '@bsv/sdk' +import { PublicKey, SignActionSpend, Transaction } from '@bsv/sdk' import { PendingSignAction, Wallet } from '../../Wallet' import { WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' import { asBsvSdkScript } from '../../utility/utilityHelpers' -import { ScriptTemplateBRC29 } from '../../utility/ScriptTemplateBRC29' +import { brc29ProtocolID, ScriptTemplateBRC29 } from '../../utility/ScriptTemplateBRC29' export async function completeSignedTransaction( prior: PendingSignAction, @@ -35,20 +35,82 @@ export async function completeSignedTransaction( /// ////////////////// // Insert SABPPP unlock templates for wallet signed inputs /// ////////////////// - for (const pdi of prior.pdi) { - const sabppp = new ScriptTemplateBRC29({ - derivationPrefix: pdi.derivationPrefix, - derivationSuffix: pdi.derivationSuffix, - keyDeriver: wallet.keyDeriver + const prepareUnlockingTemplates = ( + keys: ReturnType + ): void => { + const counterparties = new Map() + const counterparty = (publicKey: string): PublicKey => { + let parsed = counterparties.get(publicKey) + if (parsed == null) { + parsed = PublicKey.fromString(publicKey) + counterparties.set(publicKey, parsed) + } + return parsed + } + const prepared = prior.pdi.map(pdi => { + const template = new ScriptTemplateBRC29({ + derivationPrefix: pdi.derivationPrefix, + derivationSuffix: pdi.derivationSuffix, + keyDeriver: wallet.keyDeriver + }) + const unlockerPubKey = counterparty(pdi.unlockerPubKey || keys.publicKey) + return { pdi, template, unlockerPubKey } }) - const keys = wallet.getClientChangeKeyPair() - const lockerPrivKey = keys.privateKey - const unlockerPubKey = pdi.unlockerPubKey || keys.publicKey - const sourceSatoshis = pdi.sourceSatoshis - const lockingScript = asBsvSdkScript(pdi.lockingScript) - const unlockTemplate = sabppp.unlock(lockerPrivKey, unlockerPubKey, sourceSatoshis, lockingScript) - const input = prior.tx.inputs[pdi.vin] - input.unlockingScriptTemplate = unlockTemplate + const derivations = prepared.map(({ template, unlockerPubKey }) => ({ + protocolID: brc29ProtocolID, + keyID: template.getKeyID(), + counterparty: unlockerPubKey + })) + const derivedPrivateKeys = wallet.keyDeriver.derivePrivateKeys?.(derivations) ?? + derivations.map(derivation => wallet.keyDeriver.derivePrivateKey( + derivation.protocolID, + derivation.keyID, + derivation.counterparty + )) + for (let index = 0; index < prepared.length; index++) { + const { pdi, template } = prepared[index] + const unlockTemplate = template.unlockWithDerivedPrivateKey( + derivedPrivateKeys[index], + pdi.sourceSatoshis, + asBsvSdkScript(pdi.lockingScript) + ) + const input = prior.tx.inputs[pdi.vin] + input.unlockingScriptTemplate = unlockTemplate + } + } + + if (wallet.telemetry.enabled && prior.pdi.length > 0) { + await wallet.telemetry.withSpan( + 'wallet.crypto.prepare_unlocking_templates', + { + component: 'wallet-toolbox', + carrier: prior.args, + attributes: { + 'crypto.managed_input_count': prior.pdi.length + } + }, + async span => { + // Computing the root public key requires an elliptic-curve + // multiplication. It is invariant for the complete action and must not + // be repeated for every managed input in fragmented funding. + const keys = await wallet.telemetry.withSpan( + 'wallet.crypto.client_change_key', + { component: 'wallet-toolbox', parent: span.context }, + () => wallet.getClientChangeKeyPair() + ) + await wallet.telemetry.withSpan( + 'wallet.crypto.derive_unlocking_templates', + { + component: 'wallet-toolbox', + parent: span.context, + attributes: { 'crypto.managed_input_count': prior.pdi.length } + }, + () => prepareUnlockingTemplates(keys) + ) + } + ) + } else if (prior.pdi.length > 0) { + prepareUnlockingTemplates(wallet.getClientChangeKeyPair()) } /// ////////////////// diff --git a/packages/wallet/wallet-toolbox/src/signer/methods/createAction.ts b/packages/wallet/wallet-toolbox/src/signer/methods/createAction.ts index 463a53c95..0f2054a49 100644 --- a/packages/wallet/wallet-toolbox/src/signer/methods/createAction.ts +++ b/packages/wallet/wallet-toolbox/src/signer/methods/createAction.ts @@ -93,15 +93,23 @@ async function createActionCore( logger?.log('completed signed transaction') r.txid = prior.tx.id('hex') - const beef = new Beef() - if (prior.dcr.inputBeef != null) { - const inputBeef = - prior.dcr.inputBeef instanceof Uint8Array - ? Beef.fromBinaryView(prior.dcr.inputBeef) - : Beef.fromBinary(prior.dcr.inputBeef) - beef.mergeBeef(inputBeef) - } - beef.mergeTransaction(prior.tx) + const beef = await traceActionStep( + wallet, + 'wallet.create_action.assemble_result_beef', + parent, + () => { + const result = new Beef() + if (prior!.dcr.inputBeef != null) { + const inputBeef = + prior!.dcr.inputBeef instanceof Uint8Array + ? Beef.fromBinaryView(prior!.dcr.inputBeef) + : Beef.fromBinary(prior!.dcr.inputBeef) + result.mergeBeef(inputBeef) + } + result.mergeTransaction(prior!.tx) + return result + } + ) logger?.log('merged beef') await traceActionStep( @@ -115,7 +123,14 @@ async function createActionCore( r.noSendChange = prior.dcr.noSendChangeOutputVouts?.map(vout => `${r.txid}.${vout}`) beef.atomicTxid = r.txid setResultBeef(r, beef) - if (!vargs.options.returnTXIDOnly) r.tx = beef.toUint8ArrayAtomic(r.txid) + if (!vargs.options.returnTXIDOnly) { + r.tx = await traceActionStep( + wallet, + 'wallet.create_action.serialize_result_beef', + parent, + () => beef.toUint8ArrayAtomic(r.txid!) + ) + } } const { sendWithResults, notDelayedResults } = await traceActionStep( diff --git a/packages/wallet/wallet-toolbox/src/signer/methods/verifyUnlockScripts.ts b/packages/wallet/wallet-toolbox/src/signer/methods/verifyUnlockScripts.ts index 565162eb2..06f5bfbe4 100644 --- a/packages/wallet/wallet-toolbox/src/signer/methods/verifyUnlockScripts.ts +++ b/packages/wallet/wallet-toolbox/src/signer/methods/verifyUnlockScripts.ts @@ -1,8 +1,12 @@ import { Beef, + type BdkVerifierInterface, + type DigestVerification, + Hash, ScriptEvaluationError, Spend, Transaction, + TransactionSignature, type SignatureHashCache, type SpendVerificationContext, type SpendVerifierInterface @@ -14,13 +18,29 @@ export interface UnlockScriptVerificationResult { skippedInputs: number } -interface PendingSpendVerification { - inputIndex: number - resultIndex: number - spend: Spend +type PendingSpendVerification = readonly [ + inputIndex: number, + resultIndex: number, + spend: Spend, context: SpendVerificationContext +] + +type PendingTransactionVerification = readonly [ + resultIndex: number, + params: Parameters[0] +] + +type PendingDigestVerification = readonly [resultIndex: number, items: DigestVerification[]] + +interface DigestBatchVerifier { + isReady?: () => boolean + supportsCrypto?: (operation: 'verifyDigestBatch') => boolean + verifyDigestBatch: (items: readonly DigestVerification[]) => Promise } +const postChronicleHeightFallback = 943816 +const canonicalP2PKHScope = TransactionSignature.SIGHASH_ALL + TransactionSignature.SIGHASH_FORKID + const javaScriptOnlyVerifier: SpendVerifierInterface = { shouldVerifySpend: () => false, verifySpend: async () => { @@ -34,15 +54,16 @@ function invalidUnlockingScript(inputIndex: number, detail?: string): WERR_INVAL } async function verifyOneSpend(pending: PendingSpendVerification, verifier?: SpendVerifierInterface): Promise { + const [inputIndex, , spend, context] = pending try { const valid = verifier === undefined - ? pending.spend.validate(pending.context) - : await pending.spend.validateWith(verifier, pending.context) - if (!valid) throw invalidUnlockingScript(pending.inputIndex) + ? spend.validate(context) + : await spend.validateWith(verifier, context) + if (!valid) throw invalidUnlockingScript(inputIndex) } catch (error: unknown) { if (error instanceof ScriptEvaluationError) { - throw invalidUnlockingScript(pending.inputIndex, error.message) + throw invalidUnlockingScript(inputIndex, error.message) } throw error } @@ -59,7 +80,7 @@ async function verifyPendingSpends( const batched: PendingSpendVerification[] = [] for (const item of pending) { - const selected = verifier.shouldVerifySpend?.(item.spend, item.context) !== false + const selected = verifier.shouldVerifySpend?.(item[2], item[3]) !== false if (selected) batched.push(item) else await verifyOneSpend(item, javaScriptOnlyVerifier) } @@ -67,10 +88,10 @@ async function verifyPendingSpends( let verdicts: boolean[] try { - verdicts = await verifier.verifySpendsBatch(batched.map(item => ({ spend: item.spend, ...item.context }))) + verdicts = await verifier.verifySpendsBatch(batched.map(item => ({ spend: item[2], ...item[3] }))) } catch (error: unknown) { if (error instanceof ScriptEvaluationError) { - throw invalidUnlockingScript(batched[0].inputIndex, error.message) + throw invalidUnlockingScript(batched[0][0], error.message) } throw error } @@ -78,18 +99,214 @@ async function verifyPendingSpends( throw new Error('Script verifier returned an invalid batch result count') } verdicts.forEach((valid, index) => { - if (!valid) throw invalidUnlockingScript(batched[index].inputIndex) + if (!valid) throw invalidUnlockingScript(batched[index][0]) }) } +function wholeTransactionVerifier( + verifier?: SpendVerifierInterface +): (SpendVerifierInterface & BdkVerifierInterface) | undefined { + const candidate = verifier as (SpendVerifierInterface & Partial) | undefined + return typeof candidate?.verifyScripts === 'function' + ? candidate as SpendVerifierInterface & BdkVerifierInterface + : undefined +} + +function digestBatchVerifier( + verifier?: SpendVerifierInterface +): DigestBatchVerifier | undefined { + const candidate = verifier as (SpendVerifierInterface & Partial) | undefined + if (typeof candidate?.verifyDigestBatch !== 'function') return undefined + if (candidate.isReady?.() === false) return undefined + if (candidate.supportsCrypto?.('verifyDigestBatch') === false) return undefined + return candidate as SpendVerifierInterface & DigestBatchVerifier +} + +function equalBytes(left: ArrayLike, right: ArrayLike): boolean { + if (left.length !== right.length) return false + for (let index = 0; index < left.length; index++) { + if (left[index] !== right[index]) return false + } + return true +} + +function isCanonicalP2PKHLock(lock: Uint8Array): boolean { + return lock.length === 25 && + lock[0] === 0x76 && + lock[1] === 0xa9 && + lock[2] === 0x14 && + lock[23] === 0x88 && + lock[24] === 0xac +} + +function parseCanonicalP2PKHUnlock( + unlock: Uint8Array, + lock: Uint8Array +): [checksig: number[], publicKey: Uint8Array, signature: TransactionSignature] | undefined { + const signatureLength = unlock[0] + if ( + signatureLength == null || + signatureLength < 9 || + signatureLength > 73 || + unlock.length !== 1 + signatureLength + 1 + 33 || + unlock[1 + signatureLength] !== 33 + ) return undefined + const checksig = Array.from(unlock.subarray(1, 1 + signatureLength)) + const publicKey = unlock.subarray(1 + signatureLength + 1) + if ( + (publicKey[0] !== 0x02 && publicKey[0] !== 0x03) || + !equalBytes(Hash.hash160(publicKey), lock.subarray(3, 23)) + ) return undefined + + let signature: TransactionSignature + try { + signature = TransactionSignature.fromChecksigFormat(checksig) + } catch { + return undefined + } + if ( + signature.scope !== canonicalP2PKHScope || + !signature.hasLowS() || + !equalBytes(signature.toChecksigFormat(), checksig) + ) return undefined + return [checksig, publicKey, signature] +} + +/** + * Recognizes only the exact canonical P2PKH shape generated by this wallet. + * Anything else retains the general-purpose script interpreter/backend path. + */ +function standardP2PKHDigests(tx: Transaction): DigestVerification[] | undefined { + const cache: SignatureHashCache = { hashOutputsSingle: new Map() } + const items: DigestVerification[] = [] + for (let inputIndex = 0; inputIndex < tx.inputs.length; inputIndex++) { + const input = tx.inputs[inputIndex] + const sourceTransaction = input.sourceTransaction + const sourceTXID = input.sourceTXID + const unlockingScript = input.unlockingScript + if (sourceTransaction == null || sourceTXID == null || unlockingScript == null) return undefined + const sourceOutput = sourceTransaction.outputs[input.sourceOutputIndex] + if (sourceOutput == null) return undefined + const lock = sourceOutput.lockingScript.toUint8Array() + if (!isCanonicalP2PKHLock(lock)) return undefined + const parsed = parseCanonicalP2PKHUnlock(unlockingScript.toUint8Array(), lock) + if (parsed == null) return undefined + const [checksig, publicKey, signature] = parsed + + const preimage = TransactionSignature.formatBytes({ + sourceTXID, + sourceOutputIndex: input.sourceOutputIndex, + sourceSatoshis: sourceOutput.satoshis ?? 0, + transactionVersion: tx.version, + otherInputs: [], + allInputs: tx.inputs, + outputs: tx.outputs, + inputIndex, + subscript: sourceOutput.lockingScript, + inputSequence: input.sequence ?? 0xffffffff, + lockTime: tx.lockTime, + scope: signature.scope, + cache + }) + items.push({ + publicKey, + digest: Uint8Array.from(Hash.hash256(preimage)), + signature: Uint8Array.from(checksig.slice(0, -1)) + }) + } + return items +} + +async function verifyStandardP2PKHDigests( + pending: PendingDigestVerification[], + verifier: DigestBatchVerifier +): Promise> { + if (pending.length === 0) return new Set() + const items = pending.flatMap(entry => entry[1]) + const verdicts = await verifier.verifyDigestBatch(items) + if (verdicts.length !== items.length) { + throw new Error('Script verifier returned an invalid digest batch result count') + } + const verified = new Set() + let offset = 0 + for (const entry of pending) { + const end = offset + entry[1].length + if (verdicts.slice(offset, end).every(Boolean)) verified.add(entry[0]) + offset = end + } + return verified +} + +function hydrateTransactionSources( + txid: string, + transactions: ReadonlyMap +): Transaction | undefined { + const tx = transactions.get(txid) + if (tx == null) throw new WERR_INVALID_PARAMETER('txid', `contained in beef, txid ${txid}`) + for (let inputIndex = 0; inputIndex < tx.inputs.length; inputIndex++) { + const input = tx.inputs[inputIndex] + if (input.sourceTXID == null) { + throw new WERR_INVALID_PARAMETER(`inputs[${inputIndex}].sourceTXID`, 'valid') + } + if (input.unlockingScript == null) { + throw new WERR_INVALID_PARAMETER(`inputs[${inputIndex}].unlockingScript`, 'valid') + } + input.sourceTransaction = transactions.get(input.sourceTXID) + if (input.sourceTransaction == null) return undefined + if (input.sourceTransaction.outputs[input.sourceOutputIndex] == null) { + throw new WERR_INVALID_PARAMETER( + `inputs[${inputIndex}].sourceOutputIndex`, + 'reference an output in the source transaction' + ) + } + } + return tx +} + +function transactionIndex( + txids: readonly string[], + beef: Beef +): Map { + // One public lookup synchronizes nested transaction mutations and rebuilds + // BEEF's internal indexes if necessary. Repeating that synchronization for + // every input turns a large fragmented action into quadratic work. + if (txids.length > 0) beef.findTxid(txids[0]) + return new Map(beef.txs.map(item => [item.txid, item.tx])) +} + +async function verifyWholeTransactions( + pending: PendingTransactionVerification[], + verifier: BdkVerifierInterface +): Promise> { + if (pending.length === 0) return new Set() + let verdicts: boolean[] + try { + verdicts = verifier.verifyScriptsBatch === undefined + ? await Promise.all(pending.map(async item => await verifier.verifyScripts(item[1]))) + : await verifier.verifyScriptsBatch(pending.map(item => item[1])) + } catch (error: unknown) { + // Whole-transaction backends cannot identify the failing input. Preserve + // the historical per-input diagnostic lane for consensus script errors; + // integration/runtime failures still propagate unchanged. + if (error instanceof ScriptEvaluationError) return new Set() + throw error + } + if (verdicts.length !== pending.length) { + throw new Error('Script verifier returned an invalid transaction batch result count') + } + // A false verdict deliberately falls through to the historical per-input + // lane so callers retain the exact failing input in their validation error. + return new Set(pending.filter((_, index) => verdicts[index]).map(item => item[0])) +} + function collectTransactionSpends( txid: string, resultIndex: number, - beef: Beef, + transactions: ReadonlyMap, result: UnlockScriptVerificationResult, pending: PendingSpendVerification[] ): void { - const tx: Transaction | undefined = beef.findTxid(txid)?.tx + const tx = transactions.get(txid) if (tx == null) throw new WERR_INVALID_PARAMETER('txid', `contained in beef, txid ${txid}`) const sigHashCache: SignatureHashCache = { hashOutputsSingle: new Map() } for (let inputIndex = 0; inputIndex < tx.inputs.length; inputIndex++) { @@ -100,7 +317,7 @@ function collectTransactionSpends( if (input.unlockingScript == null) { throw new WERR_INVALID_PARAMETER(`inputs[${inputIndex}].unlockingScript`, 'valid') } - input.sourceTransaction = beef.findTxid(input.sourceTXID)?.tx + input.sourceTransaction = transactions.get(input.sourceTXID) if (input.sourceTransaction == null) { // knownTxids may intentionally omit a source transaction. Only that // input is skipped; every source that is present is still verified. @@ -117,11 +334,10 @@ function collectTransactionSpends( const utxoHeight = input.sourceTransaction.merklePath?.blockHeight const context: SpendVerificationContext = utxoHeight === undefined ? { consensus: true } : { consensus: true, utxoHeight } - pending.push({ + pending.push([ inputIndex, resultIndex, - context, - spend: new Spend({ + new Spend({ sourceTXID: input.sourceTXID, sourceOutputIndex: input.sourceOutputIndex, lockingScript: sourceOutput.lockingScript, @@ -135,11 +351,67 @@ function collectTransactionSpends( outputs: tx.outputs, lockTime: tx.lockTime, sigHashCache - }) - }) + }), + context + ]) } } +function collectAcceleratedTransactions( + txids: readonly string[], + transactions: ReadonlyMap, + digestVerifier: DigestBatchVerifier | undefined, + enabled: boolean +): readonly [Map, PendingDigestVerification[]] { + const hydrated = new Map() + const digests: PendingDigestVerification[] = [] + if (!enabled) return [hydrated, digests] + for (let resultIndex = 0; resultIndex < txids.length; resultIndex++) { + const tx = hydrateTransactionSources(txids[resultIndex], transactions) + if (tx == null) continue + hydrated.set(resultIndex, tx) + if (digestVerifier === undefined) continue + const items = standardP2PKHDigests(tx) + if (items != null) digests.push([resultIndex, items]) + } + return [hydrated, digests] +} + +function collectWholeTransactionVerifications( + hydrated: ReadonlyMap, + digestAttempted: ReadonlySet, + verifier: (SpendVerifierInterface & BdkVerifierInterface) | undefined +): PendingTransactionVerification[] { + if (verifier === undefined) return [] + const pending: PendingTransactionVerification[] = [] + for (const [resultIndex, tx] of hydrated) { + if (digestAttempted.has(resultIndex)) continue + const params = { tx, blockHeight: postChronicleHeightFallback, consensus: true } + if (verifier.shouldVerifyScripts?.(params) === false) continue + pending.push([resultIndex, params]) + } + return pending +} + +function collectFallbackSpends( + txids: readonly string[], + transactions: ReadonlyMap, + accelerated: ReadonlySet, + results: UnlockScriptVerificationResult[] +): PendingSpendVerification[] { + const pending: PendingSpendVerification[] = [] + for (let resultIndex = 0; resultIndex < txids.length; resultIndex++) { + if (!accelerated.has(resultIndex)) { + collectTransactionSpends(txids[resultIndex], resultIndex, transactions, results[resultIndex], pending) + continue + } + const tx = transactions.get(txids[resultIndex]) + if (tx == null) throw new WERR_INVALID_PARAMETER('txid', `contained in beef, txid ${txids[resultIndex]}`) + results[resultIndex].verifiedInputs = tx.inputs.length + } + return pending +} + /** * Verifies every resolvable input from several transactions in one optional * backend batch while preserving per-transaction verification counts. @@ -150,12 +422,27 @@ export async function verifyUnlockScriptsBatch( verifier?: SpendVerifierInterface ): Promise { const results = txids.map(() => ({ verifiedInputs: 0, skippedInputs: 0 })) - const pending: PendingSpendVerification[] = [] - for (let resultIndex = 0; resultIndex < txids.length; resultIndex++) { - collectTransactionSpends(txids[resultIndex], resultIndex, beef, results[resultIndex], pending) - } + const transactions = transactionIndex(txids, beef) + const digestVerifier = digestBatchVerifier(verifier) + const wholeVerifier = wholeTransactionVerifier(verifier) + const [hydrated, digestPending] = collectAcceleratedTransactions( + txids, + transactions, + digestVerifier, + digestVerifier !== undefined || wholeVerifier !== undefined + ) + const digestAttempted = new Set(digestPending.map(item => item[0])) + const digestVerified = digestVerifier === undefined + ? new Set() + : await verifyStandardP2PKHDigests(digestPending, digestVerifier) + const wholePending = collectWholeTransactionVerifications(hydrated, digestAttempted, wholeVerifier) + const wholeVerified = wholeVerifier === undefined + ? new Set() + : await verifyWholeTransactions(wholePending, wholeVerifier) + const accelerated = new Set([...digestVerified, ...wholeVerified]) + const pending = collectFallbackSpends(txids, transactions, accelerated, results) await verifyPendingSpends(pending, verifier) - for (const item of pending) results[item.resultIndex].verifiedInputs++ + for (const item of pending) results[item[1]].verifiedInputs++ return results } diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts index 28236f0a6..248cfa846 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts @@ -131,6 +131,10 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider return true } + protected override requiresActionBatchCleanupBeforeCreateAction(): boolean { + return false + } + /** * This method must be called at least once before any other method accesses the database, * and each time the schema may have updated. @@ -331,6 +335,37 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider return r } + override async getProvenOrRawTxs(txids: string[], trx?: TrxToken): Promise> { + const results = new Map() + const unique = [...new Set(txids)] + if (unique.length === 0) return results + + const dbTrx = this.toDbTrx(['proven_txs', 'proven_tx_reqs'], 'readonly', trx) + const provenIndex = dbTrx.objectStore('proven_txs').index('txid') + const requestIndex = dbTrx.objectStore('proven_tx_reqs').index('txid') + const usableStatuses = new Set(['unsent', 'unmined', 'unconfirmed', 'sending', 'nosend', 'completed']) + await Promise.all(unique.map(async txid => { + const proven = await provenIndex.get(txid) as TableProvenTx | undefined + if (proven != null) { + results.set(txid, { proven: this.validateEntity(proven), rawTx: undefined, inputBEEF: undefined }) + return + } + const request = await requestIndex.get(txid) as TableProvenTxReq | undefined + if (request != null && usableStatuses.has(request.status)) { + const validated = this.validateEntity(request) + results.set(txid, { + proven: undefined, + rawTx: Array.from(validated.rawTx), + inputBEEF: validated.inputBEEF == null ? undefined : Array.from(validated.inputBEEF) + }) + return + } + results.set(txid, { proven: undefined, rawTx: undefined, inputBEEF: undefined }) + })) + if (trx == null) await dbTrx.done + return results + } + async getRawTxOfKnownValidTransaction( txid?: string, offset?: number, @@ -700,6 +735,7 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider args.paged?.limit, async r => { if (!matchesProvenTxPartial(r, args.partial)) return false + if (args.txids != null && args.txids.length > 0 && !args.txids.includes(r.txid)) return false if (userId !== undefined) { const txCount = await this.countTransactions({ partial: { userId, provenTxId: r.provenTxId }, trx: dbTrx }) if (txCount === 0) return false @@ -1297,11 +1333,23 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider } override async findReservedActionBatchOutputIds(outputIds: number[], trx?: TrxToken): Promise { - const tx = this.toDbTrx(['action_batch_outputs'], 'readonly', trx) + const tx = this.toDbTrx(['action_batch_outputs', 'action_batches'], 'readonly', trx) const store = tx.objectStore('action_batch_outputs') + const batchStore = tx.objectStore('action_batches') if (store.get == null) throw new WERR_INTERNAL('IndexedDB action_batch_outputs store does not support get') const reserved: number[] = [] - for (const outputId of outputIds) if ((await store.get(outputId)) != null) reserved.push(outputId) + const now = Date.now() + for (const outputId of outputIds) { + const reservation = await store.get(outputId) + if (reservation == null) continue + const batch = await batchStore.get(reservation.actionBatchId) + if ( + batch != null && + (batch.status === 'active' || batch.status === 'prepared') && + batch.expiresAt.getTime() > now && + batch.hardExpiresAt.getTime() > now + ) reserved.push(outputId) + } if (trx == null) await tx.done return reserved } diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts index b2fcea845..01f7f67cc 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts @@ -61,6 +61,7 @@ import { verifyId, verifyOne, verifyOneOrNone } from '../utility/utilityHelpers' import { EntityTimeStamp, TransactionStatus } from '../sdk/types' import { managedChangeOutputFields } from './methods/managedChange' +import type { ManagedChangeInputCandidate } from './methods/availableManagedChange' export interface StorageKnexOptions extends StorageProviderOptions { /** @@ -72,6 +73,7 @@ export interface StorageKnexOptions extends StorageProviderOptions { // Keep bulk statements below conservative SQLite/MySQL bind-parameter // ceilings. The surrounding transaction still makes a complete pack atomic. const ACTION_BATCH_BLOB_SQL_CHUNK = 500 +const OUTPUT_INSERT_SQL_CHUNK = 500 interface KnexTelemetryQuery { __knexQueryUid?: string @@ -130,6 +132,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide } protected override supportsActionBatchPersistence (): boolean { return true } + protected override requiresActionBatchCleanupBeforeCreateAction (): boolean { return false } async readSettings (): Promise { return this.validateEntity(verifyOne(await this.toDb()('settings'))) @@ -143,7 +146,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide inputBEEF: undefined } - r.proven = verifyOneOrNone(await this.findProvenTxs({ partial: { txid } })) + r.proven = verifyOneOrNone(await this.findProvenTxs({ partial: { txid }, trx })) if (r.proven == null) { const reqRawTx = verifyOneOrNone( await k('proven_tx_reqs') @@ -159,6 +162,40 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide return r } + override async getProvenOrRawTxs (txids: string[], trx?: TrxToken): Promise> { + const unique = [...new Set(txids)] + const results = new Map() + if (unique.length === 0) return results + + const proven = await this.findProvenTxs({ partial: {}, txids: unique, trx }) + const provenTxids = new Set(proven.map(transaction => transaction.txid)) + for (const transaction of proven) { + results.set(transaction.txid, { proven: transaction, rawTx: undefined, inputBEEF: undefined }) + } + + const unresolved = unique.filter(txid => !provenTxids.has(txid)) + if (unresolved.length > 0) { + const requests = await this.findProvenTxReqs({ + partial: {}, + txids: unresolved, + status: ['unsent', 'unmined', 'unconfirmed', 'sending', 'nosend', 'completed'], + trx + }) + for (const request of requests) { + results.set(request.txid, { + proven: undefined, + rawTx: Array.from(request.rawTx), + inputBEEF: request.inputBEEF == null ? undefined : Array.from(request.inputBEEF) + }) + } + } + + for (const txid of unique) { + if (!results.has(txid)) results.set(txid, { proven: undefined, rawTx: undefined, inputBEEF: undefined }) + } + return results + } + dbTypeSubstring (source: string, fromOffset: number, forLength?: number): string { if (this.dbtype === 'MySQL') return `substring(${source} from ${fromOffset} for ${String(forLength)})` return `substr(${source}, ${fromOffset}, ${String(forLength)})` @@ -378,9 +415,14 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide override async findReservedActionBatchOutputIds (outputIds: number[], trx?: TrxToken): Promise { if (outputIds.length === 0) return [] + const now = new Date() const rows = await this.toDb(trx)('action_batch_outputs') + .join('action_batches', 'action_batch_outputs.actionBatchId', 'action_batches.actionBatchId') .whereIn('outputId', outputIds) - .select('outputId') + .whereIn('action_batches.status', ['active', 'prepared']) + .where('action_batches.expiresAt', '>', now) + .where('action_batches.hardExpiresAt', '>', now) + .select('action_batch_outputs.outputId') return rows.map(r => r.outputId) } @@ -539,6 +581,18 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide return output.outputId } + override async insertOutputs(outputs: TableOutput[], trx?: TrxToken): Promise { + if (outputs.length === 0) return + const rows = await Promise.all(outputs.map(async output => { + const row = await this.validateEntityForInsert(output, trx) + if (row.outputId === 0) delete row.outputId + return row + })) + for (let offset = 0; offset < rows.length; offset += OUTPUT_INSERT_SQL_CHUNK) { + await this.toDb(trx)('outputs').insert(rows.slice(offset, offset + OUTPUT_INSERT_SQL_CHUNK)) + } + } + override async insertOutputTag (tag: TableOutputTag, trx?: TrxToken): Promise { const e = await this.validateEntityForInsert(tag, trx, undefined, ['isDeleted']) if (e.outputTagId === 0) delete e.outputTagId @@ -864,7 +918,12 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide 'undefined. ProvenTxs may not be found by merklePath value.' ) } - return this.setupQuery('proven_txs', args) + const q = this.setupQuery('proven_txs', args) + if (args.txids != null) { + const txids = [...new Set(args.txids.filter(txid => txid !== undefined))] + if (txids.length > 0) void q.whereIn('txid', txids) + } + return q } findStaleMerkleRootsQuery (args: FindStaleMerkleRootsArgs): Knex.QueryBuilder { @@ -1392,6 +1451,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide async countChangeInputs (userId: number, basketId: number, excludeSending: boolean): Promise { const status: TransactionStatus[] = ['completed', 'unproven'] if (!excludeSending) status.push('sending') + const now = new Date() const q = this.knex('outputs as o') .join('transactions as t', 'o.transactionId', 't.transactionId') .where({ @@ -1411,7 +1471,11 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide .whereNotExists(function () { void this.select(1) .from('action_batch_outputs as abo') + .join('action_batches as ab', 'abo.actionBatchId', 'ab.actionBatchId') .whereRaw('abo.outputId = o.outputId') + .whereIn('ab.status', ['active', 'prepared']) + .where('ab.expiresAt', '>', now) + .where('ab.hardExpiresAt', '>', now) }) .whereIn('t.status', status) const count = await this.getCount(q) @@ -1426,6 +1490,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide ): Promise { const statuses: TransactionStatus[] = ['completed', 'unproven'] if (!excludeSending) statuses.push('sending') + const now = new Date() const rows = await this.toDb(trx)('outputs as o') .join('transactions as t', 'o.transactionId', 't.transactionId') .where({ @@ -1445,13 +1510,55 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide .whereNotExists(function () { void this.select(1) .from('action_batch_outputs as abo') + .join('action_batches as ab', 'abo.actionBatchId', 'ab.actionBatchId') .whereRaw('abo.outputId = o.outputId') + .whereIn('ab.status', ['active', 'prepared']) + .where('ab.expiresAt', '>', now) + .where('ab.hardExpiresAt', '>', now) }) .whereIn('t.status', statuses) .select(outputColumnsWithoutLockingScript.map(column => `o.${column}`)) return this.validateEntities(rows, undefined, ['spendable', 'change']) } + override async findAvailableManagedChangeInputCandidates ( + userId: number, + basketId: number, + excludeSending: boolean, + trx?: TrxToken + ): Promise { + const statuses: TransactionStatus[] = ['completed', 'unproven'] + if (!excludeSending) statuses.push('sending') + const now = new Date() + return await this.toDb(trx)('outputs as o') + .join('transactions as t', 'o.transactionId', 't.transactionId') + .where({ + 'o.userId': userId, + 'o.basketId': basketId, + 'o.spendable': true, + 'o.type': managedChangeOutputFields.type, + 'o.change': managedChangeOutputFields.change, + 'o.providedBy': managedChangeOutputFields.providedBy, + 'o.purpose': managedChangeOutputFields.purpose + }) + .whereNull('o.spentBy') + .whereNotNull('o.derivationPrefix') + .whereNot('o.derivationPrefix', '') + .whereNotNull('o.derivationSuffix') + .whereNot('o.derivationSuffix', '') + .whereNotExists(function () { + void this.select(1) + .from('action_batch_outputs as abo') + .join('action_batches as ab', 'abo.actionBatchId', 'ab.actionBatchId') + .whereRaw('abo.outputId = o.outputId') + .whereIn('ab.status', ['active', 'prepared']) + .where('ab.expiresAt', '>', now) + .where('ab.hardExpiresAt', '>', now) + }) + .whereIn('t.status', statuses) + .select('o.outputId', 'o.transactionId', 'o.satoshis', 'o.txid', 'o.vout') + } + override async findOutputsByIds (outputIds: number[], trx?: TrxToken): Promise> { const byId: Record = {} if (outputIds.length < 1) return byId @@ -1529,6 +1636,44 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide return statuses } + override async findFundingOutputsForUpdate ( + userId: number, + outputIds: number[], + statuses: TransactionStatus[], + trx: TrxToken + ): Promise> { + const byId: Record = {} + if (outputIds.length === 0) return byId + const uniqueOutputIds = [...new Set(outputIds)].sort((a, b) => a - b) + // Lock source outputs through their primary key first. A single joined + // SELECT ... FOR UPDATE caused PXC to lock one joined row at a time and + // turned a 150-input claim into a ~220 ms statement. Transaction status + // and action-batch reservations are then checked under the same database + // transaction; action-batch reservation follows the same output-first lock + // order, so a concurrent reserver cannot pass after this claim commits. + const rows = await this.toDb(trx)('outputs as o') + .where('o.userId', userId) + .whereIn('o.outputId', uniqueOutputIds) + .select('o.*') + .forUpdate() + const transactionIds = [...new Set(rows.map(output => output.transactionId))].sort((a, b) => a - b) + const validTransactions = transactionIds.length === 0 + ? [] + : await this.toDb(trx)>('transactions') + .where('userId', userId) + .whereIn('transactionId', transactionIds) + .whereIn('status', statuses) + .select('transactionId') + .forShare() + const validTransactionIds = new Set(validTransactions.map(transaction => transaction.transactionId)) + const reservedOutputIds = new Set(await this.findReservedActionBatchOutputIds(uniqueOutputIds, trx)) + for (const output of this.validateEntities(rows, undefined, ['spendable', 'change'])) { + if (!validTransactionIds.has(output.transactionId) || reservedOutputIds.has(output.outputId)) continue + byId[output.outputId] = output + } + return byId + } + override async findOrInsertOutputBasketsBulk ( userId: number, names: string[], diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts index d7893441e..4f845e340 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts @@ -22,7 +22,7 @@ import { mergeInputBeefs, notifyTransactionsOfProof } from './storageProviderHelpers' -import { getBeefForTransaction } from './methods/getBeefForTransaction' +import { getBeefForTransaction, getBeefForTransactions } from './methods/getBeefForTransaction' import { GetReqsAndBeefDetail, GetReqsAndBeefResult, processAction } from './methods/processAction' import { attemptToPostReqsToNetwork, PostReqsToNetworkResult } from './methods/attemptToPostReqsToNetwork' import { listCertificates } from './methods/listCertificates' @@ -108,7 +108,7 @@ import { putActionBatchBlob as putBatchBlob, putActionBatchPack as putBatchPack } from './methods/actionBatchBlobs' -import { availableManagedChange } from './methods/availableManagedChange' +import { availableManagedChange, ManagedChangeInputCandidate } from './methods/availableManagedChange' export abstract class StorageProvider extends StorageReaderWriter implements WalletStorageProvider { isDirty = false @@ -170,6 +170,15 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return updated } + /** + * Insert outputs that do not need their generated ids returned to the + * caller. Engines with a multi-row insert override this common-path helper; + * the fallback preserves existing storage implementations unchanged. + */ + async insertOutputs(outputs: TableOutput[], trx?: TrxToken): Promise { + for (const output of outputs) await this.insertOutput(output, trx) + } + /** Return unreserved wallet-managed outputs eligible for automatic funding. */ async findAvailableManagedChangeInputs( userId: number, @@ -180,6 +189,23 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return await availableManagedChange(this, userId, basketId, excludeSending, trx) } + /** Read only the fields needed by the in-memory funding planner. */ + async findAvailableManagedChangeInputCandidates( + userId: number, + basketId: number, + excludeSending: boolean, + trx?: TrxToken + ): Promise { + return (await this.findAvailableManagedChangeInputs(userId, basketId, excludeSending, trx)) + .map(({ outputId, transactionId, satoshis, txid, vout }) => ({ + outputId, + transactionId, + satoshis, + txid, + vout + })) + } + /** Read the current status of a set of source transactions without loading raw transaction bytes. */ async findTransactionStatusesByIds( userId: number, @@ -194,7 +220,52 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return statuses } + /** + * Lock and return the selected funding rows whose source transaction and + * action-batch reservation state still permit allocation. + */ + async findFundingOutputsForUpdate( + userId: number, + outputIds: number[], + statuses: TransactionStatus[], + trx: TrxToken + ): Promise> { + const rows = await this.findOutputsByIds(outputIds, trx) + const reserved = new Set(await this.findReservedActionBatchOutputIds(outputIds, trx)) + const transactionIds = [...new Set(Object.values(rows).map(output => output.transactionId))] + const transactionStatuses = await this.findTransactionStatusesByIds(userId, transactionIds, trx) + const eligible: Record = {} + for (const output of Object.values(rows)) { + if ( + output.userId === userId && + !reserved.has(output.outputId) && + statuses.includes(transactionStatuses.get(output.transactionId) as TransactionStatus) + ) eligible[output.outputId] = output + } + return eligible + } + abstract getProvenOrRawTx(txid: string, trx?: TrxToken): Promise + + /** + * Resolve several transaction proofs in one storage operation when the + * backend supports it. The default preserves compatibility for custom + * providers; SQL and IndexedDB providers override this hot path. + */ + async getProvenOrRawTxs(txids: string[], trx?: TrxToken): Promise> { + const results = new Map() + const unique = [...new Set(txids)] + let cursor = 0 + await Promise.all( + Array.from({ length: Math.min(8, unique.length) }, async () => { + while (cursor < unique.length) { + const txid = unique[cursor++] + results.set(txid, await this.getProvenOrRawTx(txid, trx)) + } + }) + ) + return results + } abstract getRawTxOfKnownValidTransaction( txid?: string, offset?: number, @@ -285,6 +356,11 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return false } + /** Custom providers may require physical expiry cleanup before reservations are queried. */ + protected requiresActionBatchCleanupBeforeCreateAction(): boolean { + return true + } + async beginActionBatch(auth: AuthId, args: BeginActionBatchArgs): Promise { if (!this.supportsActionBatchPersistence()) throw new WERR_NOT_IMPLEMENTED('actionBatch capability is not available') @@ -844,7 +920,9 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal async createAction(auth: AuthId, args: Validation.ValidCreateActionArgs): Promise { if (auth.userId == null) throw new WERR_UNAUTHORIZED() - if (this.supportsActionBatchPersistence()) await cleanupExpiredActionBatches(this) + if (this.supportsActionBatchPersistence() && this.requiresActionBatchCleanupBeforeCreateAction()) { + await cleanupExpiredActionBatches(this) + } return await createAction(this, auth, args) } @@ -1009,6 +1087,10 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return beef } + async getBeefForTransactions(txids: string[], options: StorageGetBeefOptions): Promise { + return await getBeefForTransactions(this, txids, options) + } + async findMonitorEventById(id: number, trx?: TrxToken): Promise { return verifyOneOrNone(await this.findMonitorEvents({ partial: { id }, trx })) } diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts index 9ae8d728f..2f3bfc6ff 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts @@ -4,7 +4,14 @@ import { Setup } from '../../Setup' import { SetupClient } from '../../SetupClient' import { StorageIdb } from '../StorageIdb' import { StorageProvider, StorageProviderOptions } from '../StorageProvider' -import { TableOutput, TableOutputBasket, TableTransaction, TableUser } from '../schema/tables' +import { + TableOutput, + TableOutputBasket, + TableProvenTx, + TableProvenTxReq, + TableTransaction, + TableUser +} from '../schema/tables' import { TableActionBatch } from '../schema/tables/TableActionBatch' import { StorageIdbSchema } from '../schema/StorageIdbSchema' import { openDB } from 'idb' @@ -137,6 +144,69 @@ describe('StorageIdb tests', () => { } }) + test('batch-loads proven, usable raw, and missing transaction records', async () => { + const storage = await makeStorage() + try { + const now = new Date() + const provenTxid = '16'.repeat(32) + const rawTxid = '17'.repeat(32) + const rawWithoutBeefTxid = '18'.repeat(32) + const unusableTxid = '19'.repeat(32) + const proven: TableProvenTx = { + provenTxId: 0, + txid: provenTxid, + height: 900_000, + index: 0, + merklePath: [1, 2, 3], + rawTx: [4, 5, 6], + blockHash: '20'.repeat(32), + merkleRoot: '21'.repeat(32), + created_at: now, + updated_at: now + } + const request = (txid: string, status: TableProvenTxReq['status'], inputBEEF?: number[]): TableProvenTxReq => ({ + provenTxReqId: 0, + txid, + status, + attempts: 0, + notified: false, + history: '{}', + notify: '{}', + rawTx: [7, 8, 9], + inputBEEF, + created_at: now, + updated_at: now + }) + await storage.insertProvenTx(proven) + await storage.insertProvenTxReq(request(rawTxid, 'nosend', [10, 11])) + await storage.insertProvenTxReq(request(rawWithoutBeefTxid, 'unmined')) + await storage.insertProvenTxReq(request(unusableTxid, 'invalid', [12])) + + await expect(storage.getProvenOrRawTxs([])).resolves.toEqual(new Map()) + const result = await storage.getProvenOrRawTxs([ + provenTxid, + rawTxid, + rawWithoutBeefTxid, + unusableTxid, + 'ff'.repeat(32), + provenTxid + ]) + + expect(result.get(provenTxid)?.proven?.txid).toBe(provenTxid) + expect(result.get(rawTxid)).toEqual({ proven: undefined, rawTx: [7, 8, 9], inputBEEF: [10, 11] }) + expect(result.get(rawWithoutBeefTxid)).toEqual({ proven: undefined, rawTx: [7, 8, 9], inputBEEF: undefined }) + expect(result.get(unusableTxid)).toEqual({ proven: undefined, rawTx: undefined, inputBEEF: undefined }) + expect(result.get('ff'.repeat(32))).toEqual({ proven: undefined, rawTx: undefined, inputBEEF: undefined }) + + await storage.transaction(async trx => { + const inTransaction = await storage.getProvenOrRawTxs([provenTxid], trx) + expect(inTransaction.get(provenTxid)?.proven?.txid).toBe(provenTxid) + }) + } finally { + await resetStorage(storage) + } + }) + test('reviewStatus releases outputs reserved by failed transactions', async () => { const storage = await makeStorage() try { diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageProviderBatchFallbacks.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageProviderBatchFallbacks.test.ts new file mode 100644 index 000000000..77550f0be --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageProviderBatchFallbacks.test.ts @@ -0,0 +1,101 @@ +import { StorageProvider } from '../StorageProvider' +import { ProvenOrRawTx, TrxToken } from '../../sdk/WalletStorage.interfaces' +import { TableOutput } from '../schema/tables' + +describe('StorageProvider batch fallbacks', () => { + const trx = {} as TrxToken + + test('inserts output batches in order through custom providers', async () => { + const insertOutput = jest.fn(async () => 1) + const outputs = [{ outputId: 1 }, { outputId: 2 }] as TableOutput[] + + await StorageProvider.prototype.insertOutputs.call({ insertOutput }, outputs, trx) + + expect(insertOutput.mock.calls).toEqual([ + [outputs[0], trx], + [outputs[1], trx] + ]) + }) + + test('projects only the fields required by the funding planner', async () => { + const rows = [{ + outputId: 11, + transactionId: 22, + satoshis: 33, + txid: '44'.repeat(32), + vout: 5, + lockingScript: [0x51] + }] as TableOutput[] + const findAvailableManagedChangeInputs = jest.fn(async () => rows) + + const result = await StorageProvider.prototype.findAvailableManagedChangeInputCandidates.call( + { findAvailableManagedChangeInputs }, + 1, + 2, + true, + trx + ) + + expect(result).toEqual([{ + outputId: 11, + transactionId: 22, + satoshis: 33, + txid: '44'.repeat(32), + vout: 5 + }]) + expect(findAvailableManagedChangeInputs).toHaveBeenCalledWith(1, 2, true, trx) + }) + + test('filters locked funding rows by owner, reservation, and source status', async () => { + const eligible = { outputId: 1, userId: 7, transactionId: 11 } + const reserved = { outputId: 2, userId: 7, transactionId: 12 } + const wrongUser = { outputId: 3, userId: 8, transactionId: 13 } + const wrongStatus = { outputId: 4, userId: 7, transactionId: 14 } + const rows = { 1: eligible, 2: reserved, 3: wrongUser, 4: wrongStatus } as Record + const provider = { + findOutputsByIds: jest.fn(async () => rows), + findReservedActionBatchOutputIds: jest.fn(async () => [2]), + findTransactionStatusesByIds: jest.fn(async () => new Map([ + [11, 'completed'], + [12, 'completed'], + [13, 'completed'], + [14, 'failed'] + ])) + } + + const result = await StorageProvider.prototype.findFundingOutputsForUpdate.call( + provider, + 7, + [1, 2, 3, 4], + ['completed'], + trx + ) + + expect(result).toEqual({ 1: eligible }) + expect(provider.findTransactionStatusesByIds).toHaveBeenCalledWith( + 7, + [11, 12, 13, 14], + trx + ) + }) + + test('deduplicates and concurrently resolves proof records for custom providers', async () => { + const getProvenOrRawTx = jest.fn(async (txid: string): Promise => ({ + rawTx: [Number.parseInt(txid.slice(0, 2), 16)] + })) + const provider = { getProvenOrRawTx } + + await expect(StorageProvider.prototype.getProvenOrRawTxs.call(provider, [], trx)) + .resolves.toEqual(new Map()) + const result = await StorageProvider.prototype.getProvenOrRawTxs.call( + provider, + ['11'.repeat(32), '22'.repeat(32), '11'.repeat(32)], + trx + ) + + expect([...result.keys()]).toEqual(['11'.repeat(32), '22'.repeat(32)]) + expect(getProvenOrRawTx).toHaveBeenCalledTimes(2) + expect(getProvenOrRawTx).toHaveBeenCalledWith('11'.repeat(32), trx) + expect(getProvenOrRawTx).toHaveBeenCalledWith('22'.repeat(32), trx) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts index 9cbc41e01..b1863bdd6 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts @@ -29,7 +29,7 @@ describe('createAction funding performance', () => { } ctx.activeStorage.knex.on('query', countTransactions) const markSpent = jest.spyOn(ctx.activeStorage, 'markChangeInputsSpent') - const readStatuses = jest.spyOn(ctx.activeStorage, 'findTransactionStatusesByIds') + const lockEligible = jest.spyOn(ctx.activeStorage, 'findFundingOutputsForUpdate') try { const result = await ctx.activeStorage.createAction( { userId: ctx.userId }, @@ -39,26 +39,25 @@ describe('createAction funding performance', () => { expect(result.inputs.length).toBeGreaterThan(4) expect(markSpent).toHaveBeenCalledTimes(1) expect(markSpent.mock.calls[0][0]).toHaveLength(result.inputs.length) - expect(readStatuses).toHaveBeenCalledTimes(1) - expect(readStatuses.mock.calls[0][1]).toHaveLength(1) + expect(lockEligible).toHaveBeenCalledTimes(1) + expect(lockEligible.mock.calls[0][1]).toHaveLength(result.inputs.length) expect(databaseTransactions).toBe(1) } finally { ctx.activeStorage.knex.off('query', countTransactions) } }) - test('hydrates offloaded funding scripts once per source and outside the locked lookup', async () => { + test('hydrates offloaded funding scripts once per source through the claim transaction', async () => { await replaceFundingCandidates(20, 1_000, true) - const lockedLookup = jest.spyOn(ctx.activeStorage, 'findOutputsByOutpointsForUpdate') + const lockedLookup = jest.spyOn(ctx.activeStorage, 'findFundingOutputsForUpdate') const getRawTx = jest.spyOn(ctx.activeStorage, 'getRawTxOfKnownValidTransaction') const result = await ctx.activeStorage.createAction({ userId: ctx.userId }, actionArgs(5_000)) expect(result.inputs.length).toBeGreaterThan(4) expect(lockedLookup).toHaveBeenCalledTimes(1) - expect(lockedLookup.mock.calls[0][3]).toBe(true) expect(getRawTx).toHaveBeenCalledTimes(1) - expect(getRawTx.mock.calls[0]).toEqual([expect.any(String)]) + expect(getRawTx.mock.calls[0]).toEqual([expect.any(String), undefined, undefined, expect.anything()]) }) test('rolls back a concurrent bulk-claim conflict before replanning', async () => { diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts index 53145e848..0659af729 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts @@ -4,11 +4,14 @@ import { ListOutputsResult, MerklePath, Script, + Telemetry, + TelemetryEvent, Transaction, Utils, Validation } from '@bsv/sdk' import { StorageAdminStats, StorageProvider } from '../StorageProvider' +import { EntityProvenTx } from '../schema/entities/EntityProvenTx' import { Chain } from '../../sdk/types' import { Services } from '../../services/Services' import { BlockHeader, GetMerklePathResult, GetRawTxResult } from '../../sdk/WalletServices.interfaces' @@ -139,6 +142,359 @@ describe('getBeefForTransaction tests', () => { ) }) + test('batch-loads same-block proven roots with sequentially equivalent BEEF bytes', async () => { + const storage = new ProtoStorage('main') + const transactions = Array.from({ length: 8 }, (_, index) => { + const tx = new Transaction() + tx.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromHex(`04${(500 + index).toString(16).padStart(8, '0')}`) + }) + tx.addOutput({ satoshis: index + 1, lockingScript: Script.fromASM('OP_TRUE') }) + return tx + }) + const txids = transactions.map(transaction => transaction.id('hex')) + const compound = new MerklePath(900_200, [ + txids.map((hash, offset) => ({ offset, hash, txid: true })) + ]) + const merkleRoot = compound.computeRoot(txids[0]) + const proven = new Map(transactions.map((transaction, index) => { + const now = new Date() + return [txids[index], { + proven: { + provenTxId: index + 1, + txid: txids[index], + height: compound.blockHeight, + index, + merklePath: compound.extract([txids[index]]).toBinary(), + rawTx: transaction.toBinary(), + blockHash: '11'.repeat(32), + merkleRoot, + created_at: now, + updated_at: now + } + }] + })) + const batchRead = jest.spyOn(storage, 'getProvenOrRawTxs').mockImplementation(async requested => + new Map(requested.map(txid => [txid, proven.get(txid)!])) + ) + + const actual = await storage.getBeefForTransactions(txids, { + ignoreStorage: false, + ignoreServices: true + }) + const expected = new Beef() + for (let index = 0; index < transactions.length; index++) { + expected.mergeRawTx(transactions[index].toBinary()) + expected.mergeBump(compound.extract([txids[index]])) + } + + expect(batchRead).toHaveBeenCalledTimes(1) + expect(batchRead).toHaveBeenCalledWith(txids) + expect(actual.toBinary()).toEqual(expected.toBinary()) + expect(actual.isValid()).toBe(true) + }) + + test('reports proof decode and merge failures on their batch spans', async () => { + const events: TelemetryEvent[] = [] + const storage = new ProtoStorage('main') + Object.defineProperty(storage, 'telemetry', { + value: new Telemetry({ sink: { capture: event => events.push(event) } }) + }) + const now = new Date() + const invalidTxid = '29'.repeat(32) + const invalidProven: TableProvenTx = { + provenTxId: 29, + txid: invalidTxid, + height: 900_250, + index: 0, + merklePath: [255], + rawTx: [1], + blockHash: '2a'.repeat(32), + merkleRoot: '2b'.repeat(32), + created_at: now, + updated_at: now + } + jest.spyOn(storage, 'getProvenOrRawTxs').mockResolvedValueOnce( + new Map([[invalidTxid, { proven: invalidProven }]]) + ) + const decodeFailure = jest.spyOn(EntityProvenTx.prototype, 'getMerklePath') + .mockImplementationOnce(() => { throw new Error('forced proof decode failure') }) + + await expect(storage.getBeefForTransactions([invalidTxid], { + ignoreStorage: false, + ignoreServices: true + })).rejects.toThrow('forced proof decode failure') + decodeFailure.mockRestore() + + const transaction = new Transaction() + transaction.addOutput({ satoshis: 1, lockingScript: Script.fromASM('OP_TRUE') }) + const txid = transaction.id('hex') + const path = new MerklePath(900_251, [[{ offset: 0, hash: txid, txid: true }]]) + const proven: TableProvenTx = { + provenTxId: 30, + txid, + height: path.blockHeight, + index: 0, + merklePath: path.toBinary(), + rawTx: transaction.toBinary(), + blockHash: '2c'.repeat(32), + merkleRoot: path.computeRoot(txid), + created_at: now, + updated_at: now + } + jest.spyOn(storage, 'getProvenOrRawTxs').mockResolvedValueOnce( + new Map([[txid, { proven }]]) + ) + const target = new Beef() + jest.spyOn(target, 'mergeProvenTxs').mockImplementation(() => { + throw new Error('forced batch merge failure') + }) + + await expect(storage.getBeefForTransactions([txid], { + ignoreStorage: false, + ignoreServices: true, + mergeToBeef: target + })).rejects.toThrow('forced batch merge failure') + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: 'wallet.storage.beef.decode_proven_batch', spanStatus: 'error' }), + expect.objectContaining({ name: 'wallet.storage.beef.merge_proven_batch', spanStatus: 'error' }) + ])) + }) + + test('preserves merge targets and routes proof-policy options through the single-root lane', async () => { + const storage = new ProtoStorage('main') + const roots = ['31'.repeat(32), '32'.repeat(32)] + const existing = new Beef() + existing.mergeTxidOnly(roots[0]) + + await expect(storage.getBeefForTransactions([], { + ignoreStorage: false, + ignoreServices: true, + mergeToBeef: existing + })).resolves.toBe(existing) + + const serialized = existing.toBinary() + const mergedSerialized = await storage.getBeefForTransactions(roots, { + ignoreStorage: true, + ignoreServices: true, + knownTxids: roots, + mergeToBeef: serialized, + maxConcurrency: 0 + }) + expect(mergedSerialized).not.toBe(existing) + expect(roots.every(txid => mergedSerialized.findTxid(txid) != null)).toBe(true) + + const policies: StorageGetBeefOptions[] = [ + { ignoreStorage: false, ignoreServices: true, knownTxids: roots, minProofLevel: 0 }, + { + ignoreStorage: false, + ignoreServices: true, + knownTxids: roots, + chainTracker: { + isValidRootForHeight: async () => true, + currentHeight: async () => 900_000 + }, + maxConcurrency: Number.POSITIVE_INFINITY + }, + { ignoreStorage: false, ignoreServices: true, knownTxids: roots, skipInvalidProofs: true } + ] + for (const options of policies) { + const beef = await storage.getBeefForTransactions(roots, options) + expect(roots.every(txid => beef.findTxid(txid)?.isTxidOnly === true)).toBe(true) + } + }) + + test('promotes broad known-txid lookups and enforces batch recursion depth', async () => { + const storage = new ProtoStorage('main') + const roots = Array.from({ length: 5 }, (_, index) => (100 + index).toString(16).padStart(64, '0')) + const knownTxids = Array.from({ length: 65 }, (_, index) => index.toString(16).padStart(64, '0')) + knownTxids.push(...roots) + + const known = await storage.getBeefForTransactions(roots, { + ignoreStorage: false, + ignoreServices: true, + knownTxids + }) + expect(roots.every(txid => known.findTxid(txid)?.isTxidOnly === true)).toBe(true) + + const unresolvedSource = '33'.repeat(32) + const root = new Transaction() + root.addInput({ + sourceTXID: unresolvedSource, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + root.addOutput({ satoshis: 1, lockingScript: Script.fromASM('OP_TRUE') }) + storage.maxRecursionDepth = 1 + jest.spyOn(storage, 'getProvenOrRawTxs').mockImplementation(async txids => new Map( + txids.map(txid => [txid, txid === root.id('hex') ? { rawTx: root.toBinary() } : {}]) + )) + + await expect(storage.getBeefForTransactions([root.id('hex')], { + ignoreStorage: false, + ignoreServices: true + })).rejects.toThrow('Maximum BEEF depth exceeded') + }) + + test('assembles mixed proven and raw storage records and preserves trustSelf semantics', async () => { + const storage = new ProtoStorage('main') + const provenTransaction = new Transaction() + provenTransaction.addOutput({ satoshis: 2, lockingScript: Script.fromASM('OP_TRUE') }) + const provenTxid = provenTransaction.id('hex') + const path = new MerklePath(900_300, [[{ offset: 0, hash: provenTxid, txid: true }]]) + const now = new Date() + const proven: TableProvenTx = { + provenTxId: 1, + txid: provenTxid, + height: path.blockHeight, + index: 0, + merklePath: path.toBinary(), + rawTx: provenTransaction.toBinary(), + blockHash: '34'.repeat(32), + merkleRoot: path.computeRoot(provenTxid), + created_at: now, + updated_at: now + } + const rawTransaction = new Transaction() + rawTransaction.addOutput({ satoshis: 3, lockingScript: Script.fromASM('OP_TRUE') }) + const rawTxid = rawTransaction.id('hex') + const inputBEEF = new Beef() + inputBEEF.mergeTxidOnly('35'.repeat(32)) + const stored = new Map([ + [provenTxid, { proven }], + [rawTxid, { rawTx: rawTransaction.toBinary(), inputBEEF: inputBEEF.toBinary() }] + ]) + jest.spyOn(storage, 'getProvenOrRawTxs').mockImplementation(async txids => new Map( + txids.map(txid => [txid, stored.get(txid) ?? {}]) + )) + + const assembled = await storage.getBeefForTransactions([provenTxid, rawTxid], { + ignoreStorage: false, + ignoreServices: true + }) + expect(assembled.findTxid(provenTxid)?.tx).toBeDefined() + expect(assembled.findTxid(rawTxid)?.tx).toBeDefined() + expect(assembled.findTxid('35'.repeat(32))?.isTxidOnly).toBe(true) + + const trusted = await storage.getBeefForTransactions([provenTxid, rawTxid], { + ignoreStorage: false, + ignoreServices: true, + trustSelf: 'known' + }) + expect(trusted.findTxid(provenTxid)?.isTxidOnly).toBe(true) + expect(trusted.findTxid(rawTxid)?.isTxidOnly).toBe(true) + + await expect(storage.getBeefForTransactions(['36'.repeat(32)], { + ignoreStorage: false, + ignoreServices: true + })).rejects.toThrow('valid transaction on chain main') + }) + + test('expands stored txid-only ancestors unless the current caller declares them known', async () => { + const storage = new ProtoStorage('main') + const ancestor = new Transaction() + ancestor.addOutput({ satoshis: 10, lockingScript: Script.fromASM('OP_TRUE') }) + const ancestorTxid = ancestor.id('hex') + const root = new Transaction() + root.addInput({ + sourceTXID: ancestorTxid, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + root.addOutput({ satoshis: 9, lockingScript: Script.fromASM('OP_TRUE') }) + const inheritedInputBeef = new Beef() + inheritedInputBeef.mergeTxidOnly(ancestorTxid) + const records = new Map([ + [root.id('hex'), { rawTx: root.toBinary(), inputBEEF: inheritedInputBeef.toBinary() }], + [ancestorTxid, { rawTx: ancestor.toBinary() }] + ]) + jest.spyOn(storage, 'getProvenOrRawTx').mockImplementation(async txid => records.get(txid) ?? {}) + jest.spyOn(storage, 'getProvenOrRawTxs').mockImplementation(async txids => new Map( + txids.map(txid => [txid, records.get(txid) ?? {}]) + )) + + const singleRootBeef = await storage.getBeefForTransaction(root.id('hex'), { + ignoreStorage: false, + ignoreServices: true + }) + expect(singleRootBeef.findTxid(ancestorTxid)?.tx).toBeDefined() + expect(singleRootBeef.findTxid(ancestorTxid)?.isTxidOnly).toBe(false) + + const coldClientBeef = await storage.getBeefForTransactions([root.id('hex')], { + ignoreStorage: false, + ignoreServices: true + }) + expect(coldClientBeef.findTxid(ancestorTxid)?.tx).toBeDefined() + expect(coldClientBeef.findTxid(ancestorTxid)?.isTxidOnly).toBe(false) + + const knownClientBeef = await storage.getBeefForTransactions([root.id('hex')], { + ignoreStorage: false, + ignoreServices: true, + knownTxids: [ancestorTxid] + }) + expect(knownClientBeef.findTxid(ancestorTxid)?.isTxidOnly).toBe(true) + + const knownSingleRootBeef = await storage.getBeefForTransaction(root.id('hex'), { + ignoreStorage: false, + ignoreServices: true, + knownTxids: [ancestorTxid] + }) + expect(knownSingleRootBeef.findTxid(ancestorTxid)?.isTxidOnly).toBe(true) + }) + + test('falls back to services for missing roots and to sequential proof merging for older SDK peers', async () => { + const storage = new ProtoStorage('main') + const serviceTransaction = new Transaction() + serviceTransaction.addOutput({ satoshis: 4, lockingScript: Script.fromASM('OP_TRUE') }) + const serviceTxid = serviceTransaction.id('hex') + jest.spyOn(storage, 'getProvenOrRawTxs').mockResolvedValue(new Map()) + storage.getServices().getRawTx = jest.fn(async txid => ({ + txid, + rawTx: serviceTransaction.toBinary(), + name: 'mock' + })) + storage.getServices().getMerklePath = jest.fn(async () => ({ name: 'mock' })) + + const fromServices = await storage.getBeefForTransactions([serviceTxid], { + ignoreStorage: false, + ignoreServices: false, + ignoreNewProven: true + }) + expect(fromServices.findTxid(serviceTxid)?.tx).toBeDefined() + + const provenTransaction = new Transaction() + provenTransaction.addOutput({ satoshis: 5, lockingScript: Script.fromASM('OP_TRUE') }) + const provenTxid = provenTransaction.id('hex') + const path = new MerklePath(900_400, [[{ offset: 0, hash: provenTxid, txid: true }]]) + const now = new Date() + const proven: TableProvenTx = { + provenTxId: 2, + txid: provenTxid, + height: path.blockHeight, + index: 0, + merklePath: path.toBinary(), + rawTx: provenTransaction.toBinary(), + blockHash: '37'.repeat(32), + merkleRoot: path.computeRoot(provenTxid), + created_at: now, + updated_at: now + } + jest.spyOn(storage, 'getProvenOrRawTxs').mockResolvedValue(new Map([[provenTxid, { proven }]])) + const legacyTarget = new Beef() + Object.defineProperty(legacyTarget, 'mergeProvenTxs', { value: undefined }) + + const legacy = await storage.getBeefForTransactions([provenTxid], { + ignoreStorage: false, + ignoreServices: true, + mergeToBeef: legacyTarget + }) + expect(legacy).toBe(legacyTarget) + expect(legacy.findTxid(provenTxid)?.tx).toBeDefined() + expect(legacy.isValid()).toBe(true) + }) + test('0 ProtoStorage.getBeefForTxid', async () => { const ps = new ProtoStorage('main') diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/__test/GenerateChange/generateChangeSdk.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/__test/GenerateChange/generateChangeSdk.test.ts index 14166968f..bfa87a2f9 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/__test/GenerateChange/generateChangeSdk.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/__test/GenerateChange/generateChangeSdk.test.ts @@ -204,6 +204,26 @@ describe('generateChange tests', () => { ) }) + test('2b reports the actual shortfall for a viable first change output', async () => { + const params: GenerateChangeSdkParams = { + ...defParams, + fixedInputs: [{ satoshis: 100, unlockingScriptLength: 72 }], + fixedOutputs: [{ satoshis: 90, lockingScriptLength: 25 }], + changeFirstSatoshis: 10, + targetNetCount: undefined + } + const { allocateChangeInput, releaseChangeInput } = generateChangeSdkMakeStorage([]) + + try { + await generateChangeSdk(params, allocateChangeInput, releaseChangeInput) + throw new Error('expected WERR_INSUFFICIENT_FUNDS') + } catch (error) { + expect(error).toBeInstanceOf(sdk.WERR_INSUFFICIENT_FUNDS) + expect((error as sdk.WERR_INSUFFICIENT_FUNDS).moreSatoshisNeeded).toBe(1) + expect((error as sdk.WERR_INSUFFICIENT_FUNDS).totalSatoshisNeeded).toBe(101) + } + }) + test('3 allocate all', async () => { const params: GenerateChangeSdkParams = { ...defParams, diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/__test/actionPlanning.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/__test/actionPlanning.test.ts new file mode 100644 index 000000000..fcfad271e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/__test/actionPlanning.test.ts @@ -0,0 +1,53 @@ +import { CanonicalChangeSelector, selectCanonicalChange } from '../actionPlanning' + +describe('CanonicalChangeSelector', () => { + test('matches the stateless canonical policy across allocation and release cycles', () => { + const candidates = Array.from({ length: 200 }, (_, index) => ({ + outputId: index + 1, + satoshis: ((index * 37) % 23 + 1) * 100 + })) + const selector = new CanonicalChangeSelector(candidates) + const remaining = new Set(candidates.map(candidate => candidate.outputId)) + const allocated: number[] = [] + const requests = Array.from({ length: 180 }, (_, index) => ({ + target: ((index * 61) % 29 + 1) * 90, + exact: index % 7 === 0 ? ((index * 17) % 23 + 1) * 100 : undefined + })) + + for (let index = 0; index < requests.length; index++) { + if (index > 0 && index % 11 === 0 && allocated.length > 0) { + const released = allocated.shift()! + remaining.add(released) + selector.release(released) + } + const request = requests[index] + const expected = selectCanonicalChange( + candidates.filter(candidate => remaining.has(candidate.outputId)), + request.target, + request.exact + ) + const actual = selector.take(request.target, request.exact) + expect(actual).toEqual(expected) + if (expected != null) { + remaining.delete(expected.outputId) + allocated.push(expected.outputId) + } + } + }) + + test('preserves exact, least-over, and largest-under tie ordering', () => { + const candidates = [ + { outputId: 4, satoshis: 100 }, + { outputId: 2, satoshis: 100 }, + { outputId: 3, satoshis: 200 }, + { outputId: 1, satoshis: 200 } + ] + const selector = new CanonicalChangeSelector(candidates) + + expect(selector.take(150, 100)?.outputId).toBe(2) + expect(selector.take(150)?.outputId).toBe(1) + expect(selector.take(250)?.outputId).toBe(3) + expect(selector.take(250)?.outputId).toBe(4) + expect(selector.take(1)).toBeUndefined() + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/actionPlanning.ts b/packages/wallet/wallet-toolbox/src/storage/methods/actionPlanning.ts index 2e4bbe600..5dd9680b8 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/actionPlanning.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/actionPlanning.ts @@ -26,6 +26,59 @@ export function selectCanonicalChange ( .sort((a, b) => b.satoshis - a.satoshis || b.outputId - a.outputId)[0] } +/** + * Stateful form of the canonical selector for allocating many inputs from one + * candidate set. It preserves exact / least-over / largest-under ordering but + * sorts once instead of filtering and sorting the full set per input. + */ +export class CanonicalChangeSelector { + private readonly sorted: T[] + private readonly allocated = new Set() + + constructor (outputs: readonly T[]) { + this.sorted = [...outputs].sort((a, b) => a.satoshis - b.satoshis || a.outputId - b.outputId) + } + + take (targetSatoshis: number, exactSatoshis?: number): T | undefined { + if (exactSatoshis !== undefined) { + for (let index = this.lowerBound(exactSatoshis); index < this.sorted.length; index++) { + const output = this.sorted[index] + if (output.satoshis !== exactSatoshis) break + if (!this.allocated.has(output.outputId)) return this.allocate(output) + } + } + for (let index = this.lowerBound(targetSatoshis); index < this.sorted.length; index++) { + const output = this.sorted[index] + if (!this.allocated.has(output.outputId)) return this.allocate(output) + } + for (let index = this.lowerBound(targetSatoshis) - 1; index >= 0; index--) { + const output = this.sorted[index] + if (!this.allocated.has(output.outputId)) return this.allocate(output) + } + return undefined + } + + release (outputId: number): void { + this.allocated.delete(outputId) + } + + private allocate (output: T): T { + this.allocated.add(output.outputId) + return output + } + + private lowerBound (satoshis: number): number { + let low = 0 + let high = this.sorted.length + while (low < high) { + const middle = (low + high) >>> 1 + if (this.sorted[middle].satoshis < satoshis) low = middle + 1 + else high = middle + } + return low + } +} + export function repeatableRandom (randomVals?: number[]): () => number { const values = [...(randomVals ?? [])] return () => { diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts b/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts index beb91fe08..4bcf7d052 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts @@ -4,6 +4,11 @@ import type { StorageProvider } from '../StorageProvider' import type { TableOutput } from '../schema/tables/TableOutput' import { isAutoSpendableChangeOutput, managedChangeOutputFields } from './managedChange' +export type ManagedChangeInputCandidate = Pick< + TableOutput, + 'outputId' | 'transactionId' | 'satoshis' | 'txid' | 'vout' +> + /** * Return the exact set of wallet-managed outputs currently eligible for * automatic funding. Keeping this predicate shared prevents the planner, diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts index 3ff9352e1..3ed157afb 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts @@ -14,7 +14,8 @@ import { StorageCreateTransactionSdkOutput, StorageFeeModel, StorageGetBeefOptions, - StorageProvidedBy + StorageProvidedBy, + TrxToken } from '../../sdk/WalletStorage.interfaces' import { WERR_INTERNAL, @@ -41,7 +42,8 @@ import { throwDummyReviewActions } from '../../Wallet' import { createStorageServiceChargeScript } from './offsetKey' import { WalletError } from '../../sdk' import { isAutoSpendableChangeOutput } from './managedChange' -import { randomizeOutputVouts as randomizePlannedOutputVouts, selectCanonicalChange } from './actionPlanning' +import type { ManagedChangeInputCandidate } from './availableManagedChange' +import { CanonicalChangeSelector, randomizeOutputVouts as randomizePlannedOutputVouts } from './actionPlanning' import { TransactionStatus } from '../../sdk/types' let disableDoubleSpendCheckForTest = true @@ -161,70 +163,105 @@ async function createActionCore( const initialFundingPlan = await prepareFundingPlan( storage, - { userId, vargs, xinputs, xoutputs, changeBasket, noSendChangeIn, feeModel, parent } + [userId, vargs, xinputs, xoutputs, changeBasket, noSendChangeIn, feeModel, parent] ) logger?.log(`planned funding from ${initialFundingPlan.availableChangeCount} change inputs`) + // The selected source txids are known before the write transaction begins. + // Start their one batched proof read now so PXC/network wait and BEEF + // assembly overlap the independent transaction-record/funding/output writes. + // A funding-claim retry is handled below by fetching only any residual roots. + const allocatedBeefPrefetch = startAllocatedChangeBeefPrefetch( + storage, + vargs, + initialFundingPlan.selected, + beef, + parent + ) + + const storageBeefBytes = storageBeef.toBinary() let newTx: TableTransaction | undefined + let newTxCommitted = false try { - const storageBeefBytes = storageBeef.toBinary() - newTx = await traceStorageStep( - storage, - 'wallet.storage.create_action.create_record', - parent, - { - 'action.label_count': vargs.labels.length, - 'action.storage_beef_bytes': storageBeefBytes.length - }, - async span => { - const transaction = await createNewTxRecord(storage, userId, vargs, storageBeefBytes) - span?.end({ attributes: { 'action.transaction_record_created': true } }) - return transaction + const persisted = await storage.transaction(async trx => { + const initialSatoshis = fundingPlanSatoshis(initialFundingPlan) + newTx = await traceStorageStep( + storage, + 'wallet.storage.create_action.create_record', + parent, + { + 'action.label_count': vargs.labels.length, + 'action.storage_beef_bytes': storageBeefBytes.length + }, + async span => { + const transaction = await createNewTxRecord( + storage, + userId, + vargs, + storageBeefBytes, + initialSatoshis, + trx + ) + span?.end({ attributes: { 'action.transaction_record_created': true } }) + return transaction + } + ) + logger?.log('created new transaction record') + + const ctx: CreateTransactionSdkContext = { + xinputs, + xoutputs, + changeBasket, + noSendChangeIn, + feeModel, + transactionId: newTx.transactionId } - ) - logger?.log('created new transaction record') - - const ctx: CreateTransactionSdkContext = { - xinputs, - xoutputs, - changeBasket, - noSendChangeIn, - feeModel, - transactionId: newTx.transactionId - } - const { allocatedChange, changeOutputs, derivationPrefix, maxPossibleSatoshisAdjustment } = - await fundNewTransactionSdk(storage, userId, vargs, ctx, initialFundingPlan, parent) - logger?.log('funded new transaction') + const funded = await fundNewTransactionSdk(storage, userId, vargs, ctx, initialFundingPlan, parent, trx) + logger?.log('funded new transaction') - if (maxPossibleSatoshisAdjustment != null) { - const a = maxPossibleSatoshisAdjustment - if (ctx.xoutputs[a.fixedOutputIndex].satoshis !== maxPossibleSatoshis) throw new WERR_INTERNAL() - ctx.xoutputs[a.fixedOutputIndex].satoshis = a.satoshis - logger?.log('adjusted change outputs to max possible') - } + if (funded.maxPossibleSatoshisAdjustment != null) { + const adjustment = funded.maxPossibleSatoshisAdjustment + if (ctx.xoutputs[adjustment.fixedOutputIndex].satoshis !== maxPossibleSatoshis) throw new WERR_INTERNAL() + ctx.xoutputs[adjustment.fixedOutputIndex].satoshis = adjustment.satoshis + logger?.log('adjusted change outputs to max possible') + } - // The satoshis of the transaction is the satoshis we get back in change minus the satoshis we spend. - const satoshis = - changeOutputs.reduce((a, e) => a + e.satoshis, 0) - allocatedChange.reduce((a, e) => a + e.satoshis, 0) - const { outputs, changeVouts } = await traceStorageStep( - storage, - 'wallet.storage.create_action.persist_outputs', - parent, - { - 'action.fixed_output_count': ctx.xoutputs.length, - 'action.change_output_count': changeOutputs.length - }, - async span => { - await storage.updateTransaction(newTx!.transactionId, { satoshis }) - const persisted = await createNewOutputs(storage, userId, vargs, ctx, changeOutputs) - span?.end({ attributes: { 'action.persisted_output_count': persisted.outputs.length } }) - return persisted + const satoshis = funded.changeOutputs.reduce((sum, output) => sum + output.satoshis, 0) - + funded.allocatedChange.reduce((sum, output) => sum + output.satoshis, 0) + if (satoshis !== initialSatoshis) { + await storage.updateTransaction(newTx.transactionId, { satoshis }, trx) + newTx.satoshis = satoshis } - ) + const storedOutputs = await traceStorageStep( + storage, + 'wallet.storage.create_action.persist_outputs', + parent, + { + 'action.fixed_output_count': ctx.xoutputs.length, + 'action.change_output_count': funded.changeOutputs.length + }, + async span => { + const result = await createNewOutputs(storage, userId, vargs, ctx, funded.changeOutputs, trx) + span?.end({ attributes: { 'action.persisted_output_count': result.outputs.length } }) + return result + } + ) + return { ...funded, ...storedOutputs, ctx } + }) + newTxCommitted = true + const committedTx = verifyTruthy(newTx) + const { allocatedChange, derivationPrefix, outputs, changeVouts, ctx } = persisted logger?.log('created new output records') - const inputBeef = await mergeAllocatedChangeBeefs(storage, vargs, allocatedChange, beef, parent) + const inputBeef = await mergeAllocatedChangeBeefs( + storage, + vargs, + allocatedChange, + beef, + allocatedBeefPrefetch, + parent + ) logger?.log('merged allocated change beefs') const inputs = await traceStorageStep( @@ -245,9 +282,9 @@ async function createActionCore( logger?.log('created new inputs') const r: StorageCreateActionResult = { - reference: newTx.reference, - version: newTx.version!, - lockTime: newTx.lockTime!, + reference: committedTx.reference, + version: committedTx.version!, + lockTime: committedTx.lockTime!, inputs, outputs, derivationPrefix, @@ -258,10 +295,19 @@ async function createActionCore( logger?.groupEnd() return r } catch (error) { + // Always let an overlapped database read settle before cleanup destroys or + // reuses the provider. Its error is surfaced on the success path above; + // here the construction error remains authoritative. + await allocatedBeefPrefetch if (newTx?.transactionId != null) { try { - await storage.updateTransactionStatus('failed', newTx.transactionId) - logger?.log(`marked failed createAction transaction ${newTx.transactionId} after construction error`) + if (newTxCommitted) { + await storage.updateTransactionStatus('failed', newTx.transactionId) + logger?.log(`marked failed createAction transaction ${newTx.transactionId} after construction error`) + } else { + const failed = await createNewTxRecord(storage, userId, vargs, storageBeefBytes, 0, undefined, 'failed') + logger?.log(`recorded failed createAction transaction ${failed.transactionId} after rollback`) + } } catch (cleanupError) { logger?.log(`failed to clean up createAction transaction ${newTx.transactionId}: ${String(cleanupError)}`) } @@ -469,27 +515,13 @@ async function getCompetingBeefForReview(storage: StorageProvider, txid: string) } } -/** Randomly reassign vout values across newOutputs using either the provided randomVals or crypto-random bytes. */ -/** Insert the output and attach its tags; return the SDK output descriptor. */ -async function persistNewOutput( - storage: StorageProvider, +/** Build the SDK descriptor for a persisted output. */ +function describeNewOutput( o: TableOutput, tags: string[], - txTags: Record, txBaskets: Record -): Promise<{ changeVout: number | undefined; ro: StorageCreateTransactionSdkOutput }> { - o.outputId = await storage.insertOutput(o) +): { changeVout: number | undefined; ro: StorageCreateTransactionSdkOutput } { const changeVout = o.change && o.purpose === 'change' && o.providedBy === 'storage' ? o.vout : undefined - for (const tagName of new Set(tags)) { - const tag = txTags[tagName] - await storage.insertOutputTagMap({ - outputId: verifyId(o.outputId), - outputTagId: verifyId(tag.outputTagId), - created_at: new Date(), - updated_at: new Date(), - isDeleted: false - }) - } const ro: StorageCreateTransactionSdkOutput = { vout: verifyInteger(o.vout), satoshis: Validation.validateSatoshis(o.satoshis, 'o.satoshis'), @@ -505,23 +537,47 @@ async function persistNewOutput( return { changeVout, ro } } +/** Insert the output and attach its tags; return the SDK output descriptor. */ +async function persistNewOutput( + storage: StorageProvider, + o: TableOutput, + tags: string[], + txTags: Record, + txBaskets: Record, + trx?: TrxToken +): Promise<{ changeVout: number | undefined; ro: StorageCreateTransactionSdkOutput }> { + o.outputId = await storage.insertOutput(o, trx) + for (const tagName of new Set(tags)) { + const tag = txTags[tagName] + await storage.insertOutputTagMap({ + outputId: verifyId(o.outputId), + outputTagId: verifyId(tag.outputTagId), + created_at: new Date(), + updated_at: new Date(), + isDeleted: false + }, trx) + } + return describeNewOutput(o, tags, txBaskets) +} + async function createNewOutputs( storage: StorageProvider, userId: number, vargs: Validation.ValidCreateActionArgs, ctx: CreateTransactionSdkContext, - changeOutputs: TableOutput[] + changeOutputs: TableOutput[], + trx?: TrxToken ): Promise<{ outputs: StorageCreateTransactionSdkOutput[] changeVouts: number[] }> { const txBaskets: Record = {} const basketNames = [...new Set(ctx.xoutputs.map(x => x.basket).filter((v): v is string => !!v))] - Object.assign(txBaskets, await storage.findOrInsertOutputBasketsBulk(userId, basketNames)) + Object.assign(txBaskets, await storage.findOrInsertOutputBasketsBulk(userId, basketNames, trx)) const txTags: Record = {} const tagNames = [...new Set(ctx.xoutputs.flatMap(x => x.tags))] - Object.assign(txTags, await storage.findOrInsertOutputTagsBulk(userId, tagNames)) + Object.assign(txTags, await storage.findOrInsertOutputTagsBulk(userId, tagNames, trx)) const newOutputs: Array<{ o: TableOutput; tags: string[] }> = [] @@ -539,7 +595,7 @@ async function createNewOutputs( created_at: now, updated_at: now, commissionId: 0 - }) + }, trx) const o = makeDefaultOutput(userId, ctx.transactionId, xo.satoshis, xo.vout) o.lockingScript = lockingScript o.providedBy = 'storage' @@ -571,10 +627,19 @@ async function createNewOutputs( vargs.randomVals ) + // The overwhelmingly common path has no output tags. Those rows do not + // need generated outputIds, so Knex can persist fixed and change outputs in + // one multi-row statement. Tagged rows retain the established id-dependent + // insertion path below. + const untagged = newOutputs.filter(output => output.tags.length === 0) + await storage.insertOutputs(untagged.map(output => output.o), trx) + const outputs: StorageCreateTransactionSdkOutput[] = [] const changeVouts: number[] = [] for (const { o, tags } of newOutputs) { - const { changeVout, ro } = await persistNewOutput(storage, o, tags, txTags, txBaskets) + const { changeVout, ro } = tags.length === 0 + ? describeNewOutput(o, tags, txBaskets) + : await persistNewOutput(storage, o, tags, txTags, txBaskets, trx) if (changeVout !== undefined) changeVouts.push(changeVout) outputs.push(ro) } @@ -586,7 +651,10 @@ async function createNewTxRecord( storage: StorageProvider, userId: number, vargs: Validation.ValidCreateActionArgs, - storageBeef: number[] + storageBeef: number[], + satoshis = 0, + trx?: TrxToken, + status: TableTransaction['status'] = 'unsigned' ): Promise { const now = new Date() const newTx: TableTransaction = { @@ -595,9 +663,9 @@ async function createNewTxRecord( transactionId: 0, version: vargs.version, lockTime: vargs.lockTime, - status: 'unsigned', + status, reference: randomBytesBase64(12), - satoshis: 0, // updated after fundingTransaction + satoshis, userId, isOutgoing: true, inputBEEF: storageBeef, @@ -605,13 +673,13 @@ async function createNewTxRecord( txid: undefined, rawTx: undefined } - newTx.transactionId = await storage.insertTransaction(newTx) + newTx.transactionId = await storage.insertTransaction(newTx, trx) const labelNames = [...new Set(vargs.labels)] - const labels = await storage.findOrInsertTxLabelsBulk(userId, labelNames) + const labels = await storage.findOrInsertTxLabelsBulk(userId, labelNames, trx) for (const label of labelNames) { const txLabel = labels[label] - await storage.findOrInsertTxLabelMap(verifyId(newTx.transactionId), verifyId(txLabel.txLabelId)) + await storage.findOrInsertTxLabelMap(verifyId(newTx.transactionId), verifyId(txLabel.txLabelId), trx) } return newTx @@ -919,21 +987,37 @@ async function validateNoSendChange( interface PreparedFundingPlan { params: GenerateChangeSdkParams result: GenerateChangeSdkResult - selected: TableOutput[] + selected: ManagedChangeInputCandidate[] availableChangeCount: number } -interface FundingPlanContext { - userId: number - vargs: Validation.ValidCreateActionArgs - xinputs: XValidCreateActionInput[] - xoutputs: XValidCreateActionOutput[] - changeBasket: TableOutputBasket - noSendChangeIn: TableOutput[] - feeModel: StorageFeeModel - parent?: TelemetrySpan +type FundingClaimRequest = readonly [ + userId: number, + basketId: number, + excludeSending: boolean, + transactionId: number, + noSendChangeIn: TableOutput[], + plan: PreparedFundingPlan, + trx?: TrxToken +] + +function fundingPlanSatoshis (plan: PreparedFundingPlan): number { + return plan.result.changeOutputs.reduce((sum, output) => sum + output.satoshis, 0) - + plan.selected.reduce((sum, output) => sum + output.satoshis, 0) } +type FundingPlanContext = readonly [ + userId: number, + vargs: Validation.ValidCreateActionArgs, + xinputs: XValidCreateActionInput[], + xoutputs: XValidCreateActionOutput[], + changeBasket: TableOutputBasket, + noSendChangeIn: TableOutput[], + feeModel: StorageFeeModel, + parent?: TelemetrySpan, + trx?: TrxToken +] + type FundingClaim = | { outputs: TableOutput[] @@ -1000,7 +1084,7 @@ async function prepareFundingPlan ( storage: StorageProvider, context: FundingPlanContext ): Promise { - const { userId, vargs, xinputs, xoutputs, changeBasket, noSendChangeIn, feeModel, parent } = context + const [userId, vargs, xinputs, xoutputs, changeBasket, noSendChangeIn, feeModel, parent, trx] = context const excludeSending = !vargs.isDelayed const candidates = await traceStorageStep( storage, @@ -1008,10 +1092,11 @@ async function prepareFundingPlan ( parent, { 'funding.exclude_sending': excludeSending }, async span => { - const outputs = await storage.findAvailableManagedChangeInputs( + const outputs = await storage.findAvailableManagedChangeInputCandidates( userId, changeBasket.basketId, - excludeSending + excludeSending, + trx ) span?.end({ attributes: { @@ -1037,18 +1122,16 @@ async function prepareFundingPlan ( 'funding.no_send_change_count': noSendChangeIn.length }, async span => { - const allocated = new Map() + const allocated = new Map() + const availableSelector = new CanonicalChangeSelector(available) const noSend = [...noSendChangeIn] + const noSendById = new Map(noSendChangeIn.map(output => [output.outputId, output])) const allocate = async ( targetSatoshis: number, exactSatoshis?: number ): Promise => { - let output = noSend.pop() - output ??= selectCanonicalChange( - available.filter(candidate => !allocated.has(candidate.outputId)), - targetSatoshis, - exactSatoshis - ) + let output: ManagedChangeInputCandidate | undefined = noSend.pop() + output ??= availableSelector.take(targetSatoshis, exactSatoshis) if (output == null) return undefined allocated.set(output.outputId, output) return { outputId: output.outputId, satoshis: output.satoshis } @@ -1057,7 +1140,9 @@ async function prepareFundingPlan ( const output = allocated.get(outputId) if (output == null) return allocated.delete(outputId) - if (noSendIds.has(outputId)) noSend.push(output) + availableSelector.release(outputId) + const noSendOutput = noSendById.get(outputId) + if (noSendOutput != null) noSend.push(noSendOutput) } const result = await generateChangeSdk(params, allocate, release, vargs.logger, storage.telemetry) const selected = result.allocatedChangeInputs.map(input => verifyTruthy(allocated.get(input.outputId))) @@ -1081,13 +1166,9 @@ async function prepareFundingPlan ( async function claimFundingPlan ( storage: StorageProvider, - userId: number, - basketId: number, - excludeSending: boolean, - transactionId: number, - noSendChangeIn: TableOutput[], - plan: PreparedFundingPlan + request: FundingClaimRequest ): Promise { + const [userId, basketId, excludeSending, transactionId, noSendChangeIn, plan, trx] = request if (plan.selected.length === 0) { return { outputs: [], sourceTransactionCount: 0, hydratedScriptCount: 0, scriptSourceTransactionCount: 0 } } @@ -1095,36 +1176,30 @@ async function claimFundingPlan ( const statuses: TransactionStatus[] = ['completed', 'unproven'] if (!excludeSending) statuses.push('sending') - const claim: LockedFundingClaim = await storage.transaction(async trx => { - const outpoints = plan.selected.map(output => { - if (output.txid == null) throw new WERR_INTERNAL('planned change input is missing txid') - return { txid: output.txid, vout: output.vout } - }) - const currentByOutpoint = await storage.findOutputsByOutpointsForUpdate(userId, outpoints, trx, true) - const reserved = new Set( - await storage.findReservedActionBatchOutputIds(plan.selected.map(output => output.outputId), trx) + const claim: LockedFundingClaim = await storage.transaction(async claimTrx => { + const currentById = await storage.findFundingOutputsForUpdate( + userId, + plan.selected.map(output => output.outputId), + statuses, + claimTrx ) - const transactionIds = [...new Set(Object.values(currentByOutpoint).map(output => output.transactionId))] - const transactionStatuses = await storage.findTransactionStatusesByIds(userId, transactionIds, trx) + const transactionIds = [...new Set(Object.values(currentById).map(output => output.transactionId))] const claimed: TableOutput[] = [] for (const planned of plan.selected) { - const key = `${String(planned.txid)}.${planned.vout}` - const current = currentByOutpoint[key] - const currentStatus = current == null ? undefined : transactionStatuses.get(current.transactionId) - const validTransaction = currentStatus != null && statuses.includes(currentStatus) + const current = currentById[planned.outputId] if ( current?.outputId !== planned.outputId || current?.satoshis !== planned.satoshis || current?.basketId !== basketId || !isAutoSpendableChangeOutput(current) || - reserved.has(current?.outputId ?? -1) || - validTransaction !== true + current?.txid !== planned.txid || + current?.vout !== planned.vout ) { return { conflict: noSendIds.has(planned.outputId) ? 'noSendChange' : 'candidate' } as const } claimed.push(current) } - const updated = await storage.markChangeInputsSpent(claimed.map(output => output.outputId), transactionId, trx) + const updated = await storage.markChangeInputsSpent(claimed.map(output => output.outputId), transactionId, claimTrx) if (updated !== claimed.length) { throw new FundingClaimConflict( claimed.some(output => noSendIds.has(output.outputId)) ? 'noSendChange' : 'candidate' @@ -1135,12 +1210,12 @@ async function claimFundingPlan ( output.spentBy = transactionId } return { outputs: claimed, sourceTransactionCount: transactionIds.length } - }).catch(error => { + }, trx).catch(error => { if (error instanceof FundingClaimConflict) return { conflict: error.conflict } as const throw error }) if (claim.outputs == null) return claim - const hydration = await hydrateFundingInputScripts(storage, claim.outputs) + const hydration = await hydrateFundingInputScripts(storage, claim.outputs, trx) return { outputs: claim.outputs, sourceTransactionCount: claim.sourceTransactionCount, @@ -1150,7 +1225,8 @@ async function claimFundingPlan ( async function hydrateFundingInputScripts ( storage: StorageProvider, - outputs: TableOutput[] + outputs: TableOutput[], + trx?: TrxToken ): Promise<{ hydratedScriptCount: number, scriptSourceTransactionCount: number }> { const missing = outputs.filter(output => output.lockingScript?.length !== output.scriptLength && @@ -1174,16 +1250,16 @@ async function hydrateFundingInputScripts ( while (cursor < groups.length) { const [txid, group] = groups[cursor++] if (group.length === 1) { - await storage.validateOutputScript(group[0]) + await storage.validateOutputScript(group[0], trx) continue } - const rawTx = await storage.getRawTxOfKnownValidTransaction(txid) + const rawTx = await storage.getRawTxOfKnownValidTransaction(txid, undefined, undefined, trx) if (rawTx != null) { for (const output of group) { output.lockingScript = rawTx.slice(output.scriptOffset!, output.scriptOffset! + output.scriptLength!) } } else { - for (const output of group) await storage.validateOutputScript(output) + for (const output of group) await storage.validateOutputScript(output, trx) } } }) @@ -1200,7 +1276,8 @@ async function fundNewTransactionSdk( vargs: Validation.ValidCreateActionArgs, ctx: CreateTransactionSdkContext, initialPlan: PreparedFundingPlan, - parent?: TelemetrySpan + parent?: TelemetrySpan, + trx?: TrxToken ): Promise<{ allocatedChange: TableOutput[] changeOutputs: TableOutput[] @@ -1220,15 +1297,15 @@ async function fundNewTransactionSdk( { 'funding.planned_input_count': initialPlan.selected.length }, async span => { for (let attempt = 0; attempt < 3; attempt++) { - const claim = await claimFundingPlan( - storage, + const claim = await claimFundingPlan(storage, [ userId, ctx.changeBasket.basketId, !vargs.isDelayed, ctx.transactionId, ctx.noSendChangeIn, - plan - ) + plan, + trx + ]) if (claim.outputs != null) { allocatedChange = claim.outputs span?.end({ @@ -1247,16 +1324,17 @@ async function fundNewTransactionSdk( retryCount++ plan = await prepareFundingPlan( storage, - { + [ userId, vargs, - xinputs: ctx.xinputs, - xoutputs: ctx.xoutputs, - changeBasket: ctx.changeBasket, - noSendChangeIn: ctx.noSendChangeIn, - feeModel: ctx.feeModel, - parent - } + ctx.xinputs, + ctx.xoutputs, + ctx.changeBasket, + ctx.noSendChangeIn, + ctx.feeModel, + parent, + trx + ] ) } throw new WERR_INVALID_OPERATION('wallet funding changed repeatedly during action planning; retry createAction') @@ -1375,11 +1453,84 @@ function makeKnownTxidLookup (knownTxids: string[]): (txid: string) => boolean { } } +interface AllocatedChangeBeefPrefetchResult { + beef?: Beef + error?: unknown + sourceCount: number + txids: string[] +} + +function missingAllocatedChangeTxids ( + allocatedChange: Array<{ txid?: string }>, + beef: Beef, + knownTxids: string[] +): string[] { + const hasKnownTxid = makeKnownTxidLookup(knownTxids) + return Array.from( + new Set( + allocatedChange + .map(output => verifyTruthy(output.txid)) + .filter(txid => beef.findTxid(txid) == null && !hasKnownTxid(txid)) + ) + ) +} + +function startAllocatedChangeBeefPrefetch ( + storage: StorageProvider, + vargs: Validation.ValidCreateActionArgs, + allocatedChange: ManagedChangeInputCandidate[], + beef: Beef, + parent?: TelemetrySpan +): Promise { + if (vargs.options.returnTXIDOnly) return Promise.resolve({ sourceCount: 0, txids: [] }) + const knownTxids = vargs.options.knownTxids ?? [] + const missing = missingAllocatedChangeTxids(allocatedChange, beef, knownTxids) + if (missing.length === 0) return Promise.resolve({ sourceCount: 0, txids: [] }) + const options: StorageGetBeefOptions = { + trustSelf: undefined, + knownTxids, + ignoreStorage: false, + ignoreServices: true, + ignoreNewProven: false, + minProofLevel: undefined + } + return traceStorageStep( + storage, + 'wallet.storage.create_action.beef_prefetch', + parent, + { + 'beef.planned_source_count': allocatedChange.length, + 'beef.missing_source_count': missing.length, + 'beef.storage_batch_count': missing.length === 0 ? 0 : 1 + }, + async span => { + const fetched = await storage.getBeefForTransactions(missing, options) + span?.end({ + attributes: { + 'beef.fetched_tx_count': fetched.txs.length, + 'beef.fetched_bump_count': fetched.bumps.length + } + }) + return fetched + } + ).then( + prefetched => ({ beef: prefetched, sourceCount: missing.length, txids: missing }), + error => ({ error, sourceCount: missing.length, txids: missing }) + ) +} + +function sameTxids (left: readonly string[], right: readonly string[]): boolean { + if (left.length !== right.length) return false + const expected = new Set(left) + return right.every(txid => expected.has(txid)) +} + async function mergeAllocatedChangeBeefs( storage: StorageProvider, vargs: Validation.ValidCreateActionArgs, allocatedChange: TableOutput[], beef: Beef, + prefetch: Promise, parent?: TelemetrySpan ): Promise { const options: StorageGetBeefOptions = { @@ -1393,17 +1544,29 @@ async function mergeAllocatedChangeBeefs( } if (vargs.options.returnTXIDOnly) return undefined const knownTxids = vargs.options.knownTxids ?? [] - const hasKnownTxid = makeKnownTxidLookup(knownTxids) - const missing = Array.from( - new Set( - allocatedChange - .map(output => verifyTruthy(output.txid)) - .filter(txid => beef.findTxid(txid) == null && !hasKnownTxid(txid)) - ) + const requiredBeforePrefetch = missingAllocatedChangeTxids(allocatedChange, beef, knownTxids) + const prefetched = await traceStorageStep( + storage, + 'wallet.storage.create_action.beef_prefetch_join', + parent, + { 'beef.prefetch_source_count': 0 }, + async span => { + const result = await prefetch + span?.end({ attributes: { 'beef.prefetch_source_count': result.sourceCount } }) + return result + } ) - const fetched: Array = Array.from({ length: missing.length }) - const concurrency = Math.min(8, Math.max(1, missing.length)) - let cursor = 0 + // A claim retry may replace only part of the initial funding plan. Never + // merge proofs for outputs the final transaction does not spend: apart from + // unnecessary bytes, that would disclose unrelated wallet history. + const usePrefetch = sameTxids(prefetched.txids, requiredBeforePrefetch) + if (usePrefetch && prefetched.error != null) throw prefetched.error + if (usePrefetch && prefetched.beef != null) beef.mergeBeef(prefetched.beef) + + // If a concurrent spender forced the funding claim to be replanned, only + // the newly selected roots remain. The normal uncontended path is empty. + const missing = missingAllocatedChangeTxids(allocatedChange, beef, knownTxids) + let fetched: Beef | undefined await traceStorageStep( storage, 'wallet.storage.create_action.beef_fetch', @@ -1413,21 +1576,18 @@ async function mergeAllocatedChangeBeefs( 'beef.distinct_source_count': new Set(allocatedChange.map(output => output.txid)).size, 'beef.known_txid_count': knownTxids.length, 'beef.missing_source_count': missing.length, - 'beef.fetch_concurrency': concurrency + 'beef.fetch_concurrency': 1, + 'beef.prefetch_reused': usePrefetch }, async span => { - await Promise.all( - Array.from({ length: concurrency }, async () => { - while (cursor < missing.length) { - const index = cursor++ - fetched[index] = await storage.getBeefForTransaction(missing[index], { ...options, mergeToBeef: undefined }) - } - }) - ) + if (missing.length > 0) { + fetched = await storage.getBeefForTransactions(missing, { ...options, mergeToBeef: undefined }) + } span?.end({ attributes: { - 'beef.fetched_tx_count': fetched.reduce((sum, item) => sum + (item?.txs.length ?? 0), 0), - 'beef.fetched_bump_count': fetched.reduce((sum, item) => sum + (item?.bumps.length ?? 0), 0) + 'beef.fetched_tx_count': fetched?.txs.length ?? 0, + 'beef.fetched_bump_count': fetched?.bumps.length ?? 0, + 'beef.storage_batch_count': missing.length === 0 ? 0 : 1 } }) } @@ -1436,12 +1596,9 @@ async function mergeAllocatedChangeBeefs( storage, 'wallet.storage.create_action.beef_merge', parent, - { 'beef.fragment_count': fetched.length }, + { 'beef.fragment_count': (usePrefetch && prefetched.beef != null ? 1 : 0) + (fetched == null ? 0 : 1) }, async span => { - for (const fetchedBeef of fetched) { - if (fetchedBeef == null) continue - beef.mergeBeef(fetchedBeef) - } + if (fetched != null) beef.mergeBeef(fetched) span?.end({ attributes: { 'beef.merged_tx_count': beef.txs.length, diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/generateChange.ts b/packages/wallet/wallet-toolbox/src/storage/methods/generateChange.ts index dc5c06b26..87af8dbb6 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/generateChange.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/generateChange.ts @@ -3,7 +3,7 @@ import { WalletError } from '../../sdk/WalletError' import { StorageFeeModel } from '../../sdk/WalletStorage.interfaces' import { WERR_INSUFFICIENT_FUNDS, WERR_INTERNAL, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' import { validateStorageFeeModel } from '../StorageProvider' -import { transactionSize } from './utils' +import { transactionInputSize, transactionOutputSize, transactionSize, varUintSize } from './utils' /** * An output of this satoshis amount will be adjusted to the largest fundable amount. */ @@ -254,21 +254,32 @@ async function generateChangeSdkCore( const fixedInputs = params.fixedInputs const fixedOutputs = params.fixedOutputs + const fixedFunding = fixedInputs.reduce((sum, input) => sum + input.satoshis, 0) + let fixedSpending = fixedOutputs.reduce((sum, output) => sum + output.satoshis, 0) + const fixedInputSize = fixedInputs.reduce( + (sum, input) => sum + transactionInputSize(input.unlockingScriptLength), + 0 + ) + const fixedOutputSize = fixedOutputs.reduce( + (sum, output) => sum + transactionOutputSize(output.lockingScriptLength), + 0 + ) + const changeInputSize = transactionInputSize(params.changeUnlockingScriptLength) + const changeOutputSize = transactionOutputSize(params.changeLockingScriptLength) + let allocatedFunding = 0 /** * @returns sum of transaction fixedInputs satoshis and fundingInputs satoshis */ const funding = (): number => { - return ( - fixedInputs.reduce((a, e) => a + e.satoshis, 0) + r.allocatedChangeInputs.reduce((a, e) => a + e.satoshis, 0) - ) + return fixedFunding + allocatedFunding } /** * @returns sum of transaction fixedOutputs satoshis */ const spending = (): number => { - return fixedOutputs.reduce((a, e) => a + e.satoshis, 0) + return fixedSpending } /** @@ -281,22 +292,16 @@ async function generateChangeSdkCore( const fee = (): number => funding() - spending() - change() const size = (addedChangeInputs?: number, addedChangeOutputs?: number): number => { - const inputScriptLengths = [ - ...fixedInputs.map(x => x.unlockingScriptLength), - ...Array.from( - { length: r.allocatedChangeInputs.length + (addedChangeInputs || 0) }, - () => params.changeUnlockingScriptLength - ) - ] - const outputScriptLengths = [ - ...fixedOutputs.map(x => x.lockingScriptLength), - ...Array.from( - { length: r.changeOutputs.length + (addedChangeOutputs || 0) }, - () => params.changeLockingScriptLength - ) - ] - const size = transactionSize(inputScriptLengths, outputScriptLengths) - return size + const inputCount = fixedInputs.length + r.allocatedChangeInputs.length + (addedChangeInputs || 0) + const outputCount = fixedOutputs.length + r.changeOutputs.length + (addedChangeOutputs || 0) + return 4 + + varUintSize(inputCount) + + fixedInputSize + + (r.allocatedChangeInputs.length + (addedChangeInputs || 0)) * changeInputSize + + varUintSize(outputCount) + + fixedOutputSize + + (r.changeOutputs.length + (addedChangeOutputs || 0)) * changeOutputSize + + 4 } /** @@ -349,6 +354,7 @@ async function generateChangeSdkCore( while (r.allocatedChangeInputs.length > 0) { const i = r.allocatedChangeInputs.pop() if (i != null) { + allocatedFunding -= i.satoshis await releaseChangeInput(i.outputId) } } @@ -409,6 +415,7 @@ async function generateChangeSdkCore( } r.allocatedChangeInputs.push(allocatedChangeInput) + allocatedFunding += allocatedChangeInput.satoshis maybeAddChangeOutput(ao) return true } @@ -439,6 +446,7 @@ async function generateChangeSdkCore( // resulting in pointless churn of change outputs. // And remove change inputs that funded only a single change output (along with that output)... removeChurnPairs(r.allocatedChangeInputs, r.changeOutputs) + allocatedFunding = r.allocatedChangeInputs.reduce((sum, input) => sum + input.satoshis, 0) // and try again... } } @@ -451,7 +459,9 @@ async function generateChangeSdkCore( if (feeExcess() < 0 && vgcpr.hasMaxPossibleOutput !== undefined) { // Reduce the fixed output with satoshis of maxPossibleSatoshis to what will just fund the transaction... if (fixedOutputs[vgcpr.hasMaxPossibleOutput].satoshis !== maxPossibleSatoshis) throw new WERR_INTERNAL() - fixedOutputs[vgcpr.hasMaxPossibleOutput].satoshis += feeExcess() + const adjustment = feeExcess() + fixedOutputs[vgcpr.hasMaxPossibleOutput].satoshis += adjustment + fixedSpending += adjustment r.maxPossibleSatoshisAdjustment = { fixedOutputIndex: vgcpr.hasMaxPossibleOutput, satoshis: fixedOutputs[vgcpr.hasMaxPossibleOutput].satoshis @@ -472,8 +482,11 @@ async function generateChangeSdkCore( * If needed, seek funding to avoid overspending on fees without a change output to recapture it. */ if (r.changeOutputs.length === 0 && feeExcessNow > 0) { + const minimumChange = Math.max(dustFloor, params.changeFirstSatoshis) + const totalSatoshisNeeded = spending() + feeTarget(0, 1) + minimumChange + const moreSatoshisNeeded = Math.max(1, totalSatoshisNeeded - funding()) await releaseAllocatedChangeInputs() - throw new WERR_INSUFFICIENT_FUNDS(spending() + feeTarget(), params.changeFirstSatoshis) + throw new WERR_INSUFFICIENT_FUNDS(totalSatoshisNeeded, moreSatoshisNeeded) } /** diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts index 41ef1d8dd..bceb14de0 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts @@ -4,6 +4,17 @@ import { ProvenOrRawTx, StorageGetBeefOptions } from '../../sdk/WalletStorage.in import { EntityProvenTx } from '../schema/entities/EntityProvenTx' import { WERR_INVALID_MERKLE_ROOT, WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +interface BeefFrontierItem { + txid: string + depth: number +} + +interface ProvenBeefEntry { + rawTx: number[] + merklePath: ReturnType + merkleRoot: string +} + /** * Creates a `Beef` to support the validity of a transaction identified by its `txid`. * @@ -27,14 +38,7 @@ export async function getBeefForTransaction( txid: string, options: StorageGetBeefOptions ): Promise { - let beef: Beef - if (options.mergeToBeef instanceof Beef) { - beef = options.mergeToBeef - } else if (options.mergeToBeef != null) { - beef = Beef.fromBinary(options.mergeToBeef) - } else { - beef = new Beef() - } + const beef = mergeTarget(options) // Most createAction proof requests resolve a single, already-proven root. // Building a Set for a wallet's entire known-txid history made that common @@ -44,13 +48,10 @@ export async function getBeefForTransaction( const hasKnownTxid = makeKnownTxidLookup(options.knownTxids ?? []) const scheduled = new Set([txid]) let frontier: Array<{ txid: string; depth: number }> = [{ txid, depth: 0 }] - const requestedConcurrency = options.maxConcurrency ?? 8 - const concurrency = Number.isFinite(requestedConcurrency) - ? Math.max(1, Math.min(32, Math.floor(requestedConcurrency))) - : 8 + const concurrency = normalizeConcurrency(options.maxConcurrency) while (frontier.length > 0) { - const current = frontier.filter(item => beef.findTxid(item.txid) == null) + const current = frontier.filter(item => needsResolution(beef, item.txid, hasKnownTxid)) const resolved = await mapWithConcurrency( current, concurrency, @@ -62,7 +63,7 @@ export async function getBeefForTransaction( const result = resolved[i] beef.mergeBeef(result.beef) for (const dependency of result.dependencies) { - if (!scheduled.has(dependency) && beef.findTxid(dependency) == null) { + if (!scheduled.has(dependency) && needsResolution(beef, dependency, hasKnownTxid)) { scheduled.add(dependency) next.push({ txid: dependency, depth: current[i].depth + 1 }) } @@ -74,6 +75,265 @@ export async function getBeefForTransaction( return beef } +/** + * Build one aggregate BEEF for several roots while resolving each storage + * frontier as a set. This avoids one proof query per funding input on the + * createAction success path. Complex proof-level and chain-tracker policies + * retain the established single-root implementation. + */ +export async function getBeefForTransactions( + storage: StorageProvider, + txids: string[], + options: StorageGetBeefOptions +): Promise { + const beef = mergeTarget(options) + const roots = [...new Set(txids)] + if (roots.length === 0) return beef + if (requiresSingleRootPolicy(options)) return await mergeSingleRootFragments(storage, roots, options, beef) + + const hasKnownTxid = makeKnownTxidLookup(options.knownTxids ?? []) + const scheduled = new Set(roots) + let frontier: BeefFrontierItem[] = roots.map(txid => ({ txid, depth: 0 })) + + while (frontier.length > 0) { + const unresolved = collectUnresolvedFrontier(storage, frontier, beef, hasKnownTxid) + if (unresolved.length === 0) break + + const stored = await storage.getProvenOrRawTxs(unresolved.map(item => item.txid)) + const allProven = options.trustSelf !== 'known' && unresolved.every(item => stored.get(item.txid)?.proven != null) + if (allProven) { + mergeAllProven(storage, beef, unresolved, stored) + break + } + + const [next, missing] = mergeStoredFrontier(beef, unresolved, stored, options, scheduled, hasKnownTxid) + await mergeMissingFragments(storage, beef, missing, options) + frontier = next + } + + return beef +} + +function mergeTarget(options: StorageGetBeefOptions): Beef { + if (options.mergeToBeef instanceof Beef) return options.mergeToBeef + if (options.mergeToBeef != null) return Beef.fromBinary(options.mergeToBeef) + return new Beef() +} + +function requiresSingleRootPolicy(options: StorageGetBeefOptions): boolean { + return options.ignoreStorage === true || + options.minProofLevel !== undefined || + options.chainTracker != null || + options.skipInvalidProofs === true +} + +async function mergeSingleRootFragments( + storage: StorageProvider, + roots: string[], + options: StorageGetBeefOptions, + beef: Beef +): Promise { + const fragments = await mapWithConcurrency( + roots.filter(txid => beef.findTxid(txid) == null), + normalizeConcurrency(options.maxConcurrency), + async txid => await getBeefForTransaction(storage, txid, { ...options, mergeToBeef: undefined }) + ) + for (const fragment of fragments) beef.mergeBeef(fragment) + return beef +} + +function collectUnresolvedFrontier( + storage: StorageProvider, + frontier: BeefFrontierItem[], + beef: Beef, + hasKnownTxid: (txid: string) => boolean +): BeefFrontierItem[] { + const unresolved: BeefFrontierItem[] = [] + for (const item of frontier) { + if (!needsResolution(beef, item.txid, hasKnownTxid)) continue + if (storage.maxRecursionDepth && storage.maxRecursionDepth <= item.depth) { + throw new WERR_INVALID_OPERATION(`Maximum BEEF depth exceeded. Limit is ${storage.maxRecursionDepth}`) + } + if (hasKnownTxid(item.txid)) beef.mergeTxidOnly(item.txid) + else unresolved.push(item) + } + return unresolved +} + +function decodeProvenEntries( + storage: StorageProvider, + unresolved: BeefFrontierItem[], + stored: Map +): ProvenBeefEntry[] { + const span = storage.telemetry.enabled + ? storage.telemetry.startSpan('wallet.storage.beef.decode_proven_batch', { + component: 'wallet-storage', + attributes: { 'beef.proven_tx_count': unresolved.length } + }) + : undefined + try { + const entries = unresolved.map(item => { + const proven = stored.get(item.txid)!.proven! + return { + rawTx: proven.rawTx, + merklePath: new EntityProvenTx(proven).getMerklePath(false), + merkleRoot: proven.merkleRoot + } + }) + span?.end({ attributes: { 'beef.decoded_proof_count': entries.length } }) + return entries + } catch (error) { + span?.end({ status: 'error', error }) + throw error + } +} + +function mergeAllProven( + storage: StorageProvider, + beef: Beef, + unresolved: BeefFrontierItem[], + stored: Map +): void { + const entries = decodeProvenEntries(storage, unresolved, stored) + const span = storage.telemetry.enabled + ? storage.telemetry.startSpan('wallet.storage.beef.merge_proven_batch', { + component: 'wallet-storage', + attributes: { 'beef.proven_tx_count': entries.length } + }) + : undefined + try { + mergeProvenEntries(beef, entries, unresolved, stored) + span?.end({ + attributes: { + 'beef.merged_tx_count': entries.length, + 'beef.result_tx_count': beef.txs.length, + 'beef.result_bump_count': beef.bumps.length + } + }) + } catch (error) { + span?.end({ status: 'error', error }) + throw error + } +} + +function mergeProvenEntries( + beef: Beef, + entries: ProvenBeefEntry[], + unresolved: BeefFrontierItem[], + stored: Map +): void { + if (typeof beef.mergeProvenTxs === 'function') { + beef.mergeProvenTxs(entries) + return + } + // Runtime compatibility for applications that intentionally retain an older + // compatible SDK peer. Current peers use the bulk lane; older peers retain + // the established validated sequential behavior. + for (const item of unresolved) { + const proven = stored.get(item.txid)!.proven! + beef.mergeRawTx(proven.rawTx) + beef.mergeBump(new EntityProvenTx(proven).getMerklePath()) + } +} + +function mergeStoredFrontier( + beef: Beef, + unresolved: BeefFrontierItem[], + stored: Map, + options: StorageGetBeefOptions, + scheduled: Set, + hasKnownTxid: (txid: string) => boolean +): [next: BeefFrontierItem[], missing: BeefFrontierItem[]] { + const next: BeefFrontierItem[] = [] + const missing: BeefFrontierItem[] = [] + for (const item of unresolved) { + const result = stored.get(item.txid) + if (result?.proven != null) mergeStoredProven(beef, item, result, options) + else if (result?.rawTx != null) mergeStoredRaw(beef, item, result, options, scheduled, next, hasKnownTxid) + else missing.push(item) + } + return [next, missing] +} + +function mergeStoredProven( + beef: Beef, + item: BeefFrontierItem, + result: ProvenOrRawTx, + options: StorageGetBeefOptions +): void { + if (options.trustSelf === 'known') { + beef.mergeTxidOnly(item.txid) + return + } + const proven = result.proven! + beef.mergeRawTx(proven.rawTx) + beef.mergeBump(new EntityProvenTx(proven).getMerklePath()) +} + +function mergeStoredRaw( + beef: Beef, + item: BeefFrontierItem, + result: ProvenOrRawTx, + options: StorageGetBeefOptions, + scheduled: Set, + next: BeefFrontierItem[], + hasKnownTxid: (txid: string) => boolean +): void { + if (options.trustSelf === 'known') { + beef.mergeTxidOnly(item.txid) + return + } + const transaction = beef.mergeRawTx(result.rawTx!) + if (result.inputBEEF != null) beef.mergeBeef(result.inputBEEF) + appendNewDependencies(transaction.inputTxids, item.depth + 1, beef, scheduled, next, hasKnownTxid) +} + +function appendNewDependencies( + dependencies: string[], + depth: number, + beef: Beef, + scheduled: Set, + next: BeefFrontierItem[], + hasKnownTxid: (txid: string) => boolean +): void { + for (const txid of dependencies) { + if (scheduled.has(txid) || !needsResolution(beef, txid, hasKnownTxid)) continue + scheduled.add(txid) + next.push({ txid, depth }) + } +} + +function needsResolution( + beef: Beef, + txid: string, + hasKnownTxid: (txid: string) => boolean +): boolean { + const entry = beef.findTxid(txid) + return entry == null || (entry.isTxidOnly && !hasKnownTxid(txid)) +} + +async function mergeMissingFragments( + storage: StorageProvider, + beef: Beef, + missing: BeefFrontierItem[], + options: StorageGetBeefOptions +): Promise { + if (missing.length === 0) return + if (options.ignoreServices === true) { + throw new WERR_INVALID_PARAMETER(`txid ${missing[0].txid}`, `valid transaction on chain ${storage.chain}`) + } + const fragments = await mapWithConcurrency( + missing, + normalizeConcurrency(options.maxConcurrency), + async item => await getBeefForTransaction(storage, item.txid, { + ...options, + ignoreStorage: true, + mergeToBeef: undefined + }) + ) + for (const fragment of fragments) beef.mergeBeef(fragment) +} + function makeKnownTxidLookup (knownTxids: string[]): (txid: string) => boolean { let lookups = 0 let indexed: Set | undefined @@ -88,6 +348,12 @@ function makeKnownTxidLookup (knownTxids: string[]): (txid: string) => boolean { } } +function normalizeConcurrency(value: number | undefined = 8): number { + return Number.isFinite(value) + ? Math.max(1, Math.min(32, Math.floor(value))) + : 8 +} + async function mapWithConcurrency( values: T[], concurrency: number, diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts index 971426fa5..0607410ff 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts @@ -4,7 +4,8 @@ import { Transaction as BsvTransaction, SendWithResult, SendWithResultStatus, - WalletLoggerInterface + WalletLoggerInterface, + TelemetrySpan } from '@bsv/sdk' import { aggregateActionResults } from '../../utility/aggregateResults' import { StorageProvider } from '../StorageProvider' @@ -31,7 +32,6 @@ import { ProvenTxReqStatus, TransactionStatus } from '../../sdk/types' import { parseTxScriptOffsets, TxScriptOffsets } from '../../utility/parseTxScriptOffsets' import { TableTransaction } from '../schema/tables/TableTransaction' import { TableOutput } from '../schema/tables/TableOutput' -import { TableCommission } from '../schema/tables/TableCommission' import { asArray, asString } from '../../utility/utilityHelpers.noBuffer' import { WalletError } from '../../sdk/WalletError' import { classifyReqStatus } from '../storageProviderHelpers' @@ -40,6 +40,33 @@ export async function processAction( storage: StorageProvider, auth: AuthId, args: StorageProcessActionArgs +): Promise { + if (!storage.telemetry.enabled) return await processActionCore(storage, auth, args) + return await storage.telemetry.withSpan( + 'wallet.storage.process_action', + { + component: 'wallet-storage', + carrier: args, + attributes: { + 'action.is_new_transaction': args.isNewTx, + 'action.is_no_send': args.isNoSend, + 'action.is_delayed': args.isDelayed, + 'action.send_with_count': args.sendWith.length + } + }, + async span => { + const result = await processActionCore(storage, auth, args, span) + span.end({ attributes: { 'action.send_result_count': result.sendWithResults?.length ?? 0 } }) + return result + } + ) +} + +async function processActionCore( + storage: StorageProvider, + auth: AuthId, + args: StorageProcessActionArgs, + parent?: TelemetrySpan ): Promise { const logger = args.logger logger?.group('storage processAction') @@ -53,9 +80,19 @@ export async function processAction( const txidsOfReqsToShareWithWorld: string[] = [...args.sendWith] if (args.isNewTx) { - const vargs = await validateCommitNewTxToStorageArgs(storage, userId, args) + const vargs = await traceProcessStep( + storage, + 'wallet.storage.process_action.validate', + parent, + async () => await validateCommitNewTxToStorageArgs(storage, userId, args) + ) logger?.log('validated new tx updates to storage') - ;({ req } = await commitNewTxToStorage(storage, userId, vargs)) + ;({ req } = await traceProcessStep( + storage, + 'wallet.storage.process_action.commit', + parent, + async () => await commitNewTxToStorage(storage, userId, vargs) + )) logger?.log('committed new tx updates to storage ') if (!req) throw new WERR_INTERNAL() // Add the new txid to sendWith unless there are no others to send and the noSend option is set. @@ -67,13 +104,18 @@ export async function processAction( } } - const { swr, ndr } = await shareReqsWithWorld( + const { swr, ndr } = await traceProcessStep( storage, - userId, - txidsOfReqsToShareWithWorld, - args.isDelayed, - undefined, - logger + 'wallet.storage.process_action.share', + parent, + async () => await shareReqsWithWorld( + storage, + userId, + txidsOfReqsToShareWithWorld, + args.isDelayed, + undefined, + logger + ) ) r.sendWithResults = swr @@ -84,6 +126,20 @@ export async function processAction( return r } +async function traceProcessStep( + storage: StorageProvider, + name: string, + parent: TelemetrySpan | undefined, + callback: () => Promise +): Promise { + if (parent == null) return await callback() + return await storage.telemetry.withSpan( + name, + { component: 'wallet-storage', parent: parent.context }, + callback + ) +} + export interface GetReqsAndBeefDetail { txid: string req?: TableProvenTxReq @@ -315,10 +371,7 @@ interface ValidCommitNewTxToStorageArgs { txScriptOffsets: TxScriptOffsets transactionId: number transaction: TableTransaction - inputOutputs: TableOutput[] outputOutputs: TableOutput[] - commission: TableCommission | undefined - beef: Beef req: EntityProvenTxReq outputUpdates: Array<{ id: number; update: Partial }> @@ -358,21 +411,22 @@ async function validateCommitNewTxToStorageArgs( ) if (!transaction.isOutgoing) throw new WERR_INVALID_OPERATION('isOutgoing is not true') if (transaction.inputBEEF == null) throw new WERR_INVALID_OPERATION() - const beef = Beef.fromBinary(asArray(transaction.inputBEEF)) - // Could check beef validates transaction inputs... // Transaction must have unsigned or unprocessed status if (transaction.status !== 'unsigned' && transaction.status !== 'unprocessed') { throw new WERR_INVALID_OPERATION(`invalid transaction status ${transaction.status}`) } const transactionId = verifyId(transaction.transactionId) - const outputOutputs = await storage.findOutputs({ - partial: { userId, transactionId } - }) - const inputOutputs = await storage.findOutputs({ - partial: { userId, spentBy: transactionId } - }) - - const commission = verifyOneOrNone(await storage.findCommissions({ partial: { transactionId, userId } })) + // These reads are independent once the planned transaction is resolved. + // Running them together removes two network-database round trips from every + // successful remote-storage commit. + const [outputOutputs, commissionRows] = await Promise.all([ + storage.findOutputs({ partial: { userId, transactionId } }), + storage.commissionSatoshis > 0 + ? storage.findCommissions({ partial: { transactionId, userId } }) + : Promise.resolve([]) + ]) + + const commission = verifyOneOrNone(commissionRows) if (storage.commissionSatoshis > 0) { // A commission is required... if (commission == null) throw new WERR_INTERNAL() @@ -414,10 +468,7 @@ async function validateCommitNewTxToStorageArgs( txScriptOffsets, transactionId, transaction, - inputOutputs, outputOutputs, - commission, - beef, req, outputUpdates: [], // update txid, status in transactions table and drop rawTransaction value diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/KnexSessionManager.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/KnexSessionManager.ts index e79bb116b..6317c5cf3 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/KnexSessionManager.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/KnexSessionManager.ts @@ -4,10 +4,18 @@ import { TableAuthSession, tableAuthSessionToPeerSession } from '../schema/table export const AUTH_SESSION_TABLE = 'auth_sessions' export const DEFAULT_AUTH_SESSION_TTL_MS = 24 * 60 * 60 * 1000 +export const DEFAULT_AUTH_SESSION_TOUCH_INTERVAL_MS = 60 * 1000 +type NullableBoolean = boolean | number | null | undefined export interface KnexSessionManagerOptions { /** Session lifetime since its most recent authenticated use. Default: 24 hours. */ ttlMs?: number + /** + * Maximum time that an authenticated, timestamp-only session update may be + * coalesced. Authentication and certificate state changes are always written + * immediately. Default: 1 minute (or one quarter of ttlMs when shorter). + */ + touchIntervalMs?: number /** Testable clock source. Defaults to `Date.now`. */ now?: () => number } @@ -22,37 +30,47 @@ export interface KnexSessionManagerOptions { */ export class KnexSessionManager implements AsyncSessionManager { private readonly ttlMs: number + private readonly touchIntervalMs: number private readonly now: () => number + /** Rows associated with session objects returned by this manager. */ + private readonly persistedRows = new WeakMap() constructor (private readonly knex: Knex, options: KnexSessionManagerOptions = {}) { this.ttlMs = options.ttlMs ?? DEFAULT_AUTH_SESSION_TTL_MS + this.touchIntervalMs = options.touchIntervalMs ?? Math.min( + DEFAULT_AUTH_SESSION_TOUCH_INTERVAL_MS, + Math.max(1, Math.floor(this.ttlMs / 4)) + ) this.now = options.now ?? Date.now if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs <= 0) { throw new TypeError('KnexSessionManager ttlMs must be a positive safe integer.') } + if (!Number.isSafeInteger(this.touchIntervalMs) || this.touchIntervalMs < 0) { + throw new TypeError('KnexSessionManager touchIntervalMs must be a non-negative safe integer.') + } } async addSession (session: PeerSession): Promise { - await this.persistSession(session) + await this.persistSession(session, false) } async updateSession (session: PeerSession): Promise { - await this.persistSession(session) + await this.persistSession(session, true) } async getSession (identifier: string): Promise { const byNonce = await this.activeSessions() .where({ sessionNonce: identifier }) .first() - if (byNonce != null) return tableAuthSessionToPeerSession(byNonce) + if (byNonce != null) return this.sessionForRow(byNonce) const byIdentity = await this.activeSessions() .where({ peerIdentityKey: identifier }) .orderBy('lastUpdate', 'desc') .orderBy('sessionNonce', 'desc') .first() - return byIdentity == null ? undefined : tableAuthSessionToPeerSession(byIdentity) + return byIdentity == null ? undefined : this.sessionForRow(byIdentity) } async removeSession (session: PeerSession): Promise { @@ -103,17 +121,16 @@ export class KnexSessionManager implements AsyncSessionManager { .where('expiresAt', '>', this.now()) } - private async persistSession (session: PeerSession): Promise { - if (typeof session.sessionNonce !== 'string' || session.sessionNonce.length === 0) { - throw new TypeError('Invalid session: sessionNonce is required to persist a session.') - } - if (!Number.isSafeInteger(session.lastUpdate) || session.lastUpdate < 0) { - throw new TypeError('Invalid session: lastUpdate must be a non-negative safe integer.') - } - + private async persistSession (session: PeerSession, coalesceTouch: boolean): Promise { + this.validatePersistentSession(session) const row = this.toTableAuthSession(session) + if (coalesceTouch && this.canCoalesceTouch(session, row)) return + const updated = await this.updateIfCurrentOrNewer(row) - if (updated > 0) return + if (updated > 0) { + this.persistedRows.set(session, row) + return + } // A zero-row update can mean either that this write is stale or that the // database reports no changed rows for an idempotent update. Avoid an @@ -125,21 +142,64 @@ export class KnexSessionManager implements AsyncSessionManager { if (existing != null) { // A concurrent insert may have appeared after the first update. Retry if // our state is still current enough to advance or merge that new row. - if (existing.lastUpdate <= row.lastUpdate) await this.updateIfCurrentOrNewer(row) + if (existing.lastUpdate <= row.lastUpdate) { + const retried = await this.updateIfCurrentOrNewer(row) + if (retried > 0) this.persistedRows.set(session, row) + } return } try { await this.knex(AUTH_SESSION_TABLE).insert(row) + this.persistedRows.set(session, row) } catch (error: unknown) { // Another replica may have inserted this nonce between our update and // insert. Retry only known duplicate-key races; preserve every other // database failure for the caller. if (!isDuplicateKeyError(error)) throw error - await this.updateIfCurrentOrNewer(row) + const retried = await this.updateIfCurrentOrNewer(row) + if (retried > 0) this.persistedRows.set(session, row) + } + } + + private validatePersistentSession (session: PeerSession): void { + if (typeof session.sessionNonce !== 'string' || session.sessionNonce.length === 0) { + throw new TypeError('Invalid session: sessionNonce is required to persist a session.') + } + if (!Number.isSafeInteger(session.lastUpdate) || session.lastUpdate < 0) { + throw new TypeError('Invalid session: lastUpdate must be a non-negative safe integer.') } } + /** + * Coalesce only the routine last-used write performed after an authenticated + * general message. The WeakMap proves that this exact session object came + * from a durable row read (or successful write) by this manager. Every + * authentication/certificate transition and every unrecognized object still + * takes the monotonic database path. + */ + private canCoalesceTouch (session: PeerSession, row: TableAuthSession): boolean { + if (this.touchIntervalMs === 0 || row.isAuthenticated !== true) return false + const persisted = this.persistedRows.get(session) + if (!persisted?.isAuthenticated) return false + + const persistedLastUpdate = Number(persisted.lastUpdate) + const elapsed = Number(row.lastUpdate) - persistedLastUpdate + if (elapsed < 0 || elapsed >= this.touchIntervalMs) return false + if (Number(persisted.expiresAt) - this.now() <= this.touchIntervalMs) return false + + return nullableEqual(row.peerNonce, persisted.peerNonce) && + nullableEqual(row.peerIdentityKey, persisted.peerIdentityKey) && + nullableBooleanEqual(row.certificatesRequired, persisted.certificatesRequired) && + nullableBooleanEqual(row.certificatesValidated, persisted.certificatesValidated) + } + + private sessionForRow (row: TableAuthSession): PeerSession { + const session = tableAuthSessionToPeerSession(row) + this.persistedRows.set(session, { ...row }) + return session + } + private async updateIfCurrentOrNewer (row: TableAuthSession): Promise { return await this.knex(AUTH_SESSION_TABLE) .where({ sessionNonce: row.sessionNonce }) @@ -164,7 +224,7 @@ export class KnexSessionManager implements AsyncSessionManager { }) } - private mergeNullableBoolean (column: string, value: boolean | number | null | undefined): Knex.Raw { + private mergeNullableBoolean (column: string, value: NullableBoolean): Knex.Raw { let incoming: 0 | 1 | null if (value == null) { incoming = null @@ -212,6 +272,18 @@ function applyNullableMatch ( } } +function nullableEqual (left: string | null | undefined, right: string | null | undefined): boolean { + return (left ?? null) === (right ?? null) +} + +function nullableBooleanEqual ( + left: boolean | number | null | undefined, + right: boolean | number | null | undefined +): boolean { + if (left == null || right == null) return left == null && right == null + return Boolean(left) === Boolean(right) +} + function isDuplicateKeyError (error: unknown): boolean { if (typeof error !== 'object' || error == null) return false const databaseError = error as { code?: unknown, errno?: unknown } diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexSessionManager.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexSessionManager.test.ts index 063821b6b..144db8600 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexSessionManager.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexSessionManager.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { Knex, knex as makeKnex } from 'knex' import { AUTH_SESSION_MIGRATION, KnexMigrations } from '../../schema/KnexMigrations' -import { KnexSessionManager } from '../KnexSessionManager' +import { AUTH_SESSION_TABLE, KnexSessionManager } from '../KnexSessionManager' describe('KnexSessionManager', () => { let folder: string @@ -136,8 +136,111 @@ describe('KnexSessionManager', () => { await expect(managerB.pruneExpiredSessions()).resolves.toBe(1) }) + it('coalesces only recent timestamp touches for a row-backed authenticated session', async () => { + await managerA.addSession(makeSession({ + isAuthenticated: true, + lastUpdate: 1_000 + })) + const session = await managerB.getSession('session-nonce') + expect(session).toBeDefined() + + now = 1_010 + session!.lastUpdate = now + await managerB.updateSession(session!) + await expect(knexA(AUTH_SESSION_TABLE).where({ sessionNonce: 'session-nonce' }).first()) + .resolves.toMatchObject({ lastUpdate: 1_000, expiresAt: 1_100 }) + + // The test TTL gives the default touch window a 25 ms boundary. Reaching + // that boundary refreshes the durable timestamp and expiration. + now = 1_025 + session!.lastUpdate = now + await managerB.updateSession(session!) + await expect(knexA(AUTH_SESSION_TABLE).where({ sessionNonce: 'session-nonce' }).first()) + .resolves.toMatchObject({ lastUpdate: 1_025, expiresAt: 1_125 }) + }) + + it('persists security-state transitions and supports exact timestamp persistence', async () => { + await managerA.addSession(makeSession({ + isAuthenticated: true, + certificatesRequired: true, + certificatesValidated: false + })) + const session = await managerB.getSession('session-nonce') + expect(session).toBeDefined() + + now = 1_005 + session!.lastUpdate = now + session!.certificatesValidated = true + await managerB.updateSession(session!) + await expect(knexA(AUTH_SESSION_TABLE).where({ sessionNonce: 'session-nonce' }).first()) + .resolves.toMatchObject({ lastUpdate: 1_005, certificatesValidated: 1 }) + + const exactManager = new KnexSessionManager(knexB, { + ttlMs: 100, + touchIntervalMs: 0, + now: () => now + }) + const exactSession = await exactManager.getSession('session-nonce') + expect(exactSession).toBeDefined() + now = 1_006 + exactSession!.lastUpdate = now + await exactManager.updateSession(exactSession!) + await expect(knexA(AUTH_SESSION_TABLE).where({ sessionNonce: 'session-nonce' }).first()) + .resolves.toMatchObject({ lastUpdate: 1_006, expiresAt: 1_106 }) + }) + + it('retries a current session update after a concurrent insert becomes visible', async () => { + const session = makeSession() + await managerA.addSession(session) + const update = jest.spyOn(managerA as any, 'updateIfCurrentOrNewer') + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(1) + + await managerA.addSession(session) + + expect(update).toHaveBeenCalledTimes(2) + + update.mockClear() + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(0) + await managerA.addSession(session) + expect(update).toHaveBeenCalledTimes(2) + }) + + it('recovers from a duplicate-key insert race and preserves other insert failures', async () => { + const duplicateInsert = jest.fn(async () => await Promise.reject( + Object.assign(new Error('duplicate'), { code: 'ER_DUP_ENTRY' }) + )) + const failingInsert = jest.fn(async () => await Promise.reject(new Error('database unavailable'))) + const makeKnex = (insert: jest.Mock): Knex => { + const first = jest.fn(async () => undefined) + const where = jest.fn(() => ({ first })) + return jest.fn(() => ({ where, insert })) as unknown as Knex + } + + const duplicateManager = new KnexSessionManager(makeKnex(duplicateInsert)) + const duplicateUpdate = jest.spyOn(duplicateManager as any, 'updateIfCurrentOrNewer') + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(1) + await duplicateManager.addSession(makeSession()) + expect(duplicateUpdate).toHaveBeenCalledTimes(2) + + const unchangedManager = new KnexSessionManager(makeKnex(duplicateInsert)) + const unchangedUpdate = jest.spyOn(unchangedManager as any, 'updateIfCurrentOrNewer') + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(0) + await unchangedManager.addSession(makeSession()) + expect(unchangedUpdate).toHaveBeenCalledTimes(2) + + const failingManager = new KnexSessionManager(makeKnex(failingInsert)) + jest.spyOn(failingManager as any, 'updateIfCurrentOrNewer').mockResolvedValueOnce(0) + await expect(failingManager.addSession(makeSession())).rejects.toThrow('database unavailable') + }) + it('rejects invalid options and session records', async () => { expect(() => new KnexSessionManager(knexA, { ttlMs: 0 })).toThrow('ttlMs') + expect(() => new KnexSessionManager(knexA, { touchIntervalMs: -1 })).toThrow('touchIntervalMs') + expect(() => new KnexSessionManager(knexA, { touchIntervalMs: 1.5 })).toThrow('touchIntervalMs') await expect(managerA.addSession(makeSession({ sessionNonce: undefined }))).rejects.toThrow('sessionNonce') await expect(managerA.addSession(makeSession({ lastUpdate: Number.NaN }))).rejects.toThrow('lastUpdate') await expect(managerA.addSession(makeSession({ lastUpdate: Number.MAX_SAFE_INTEGER }))).rejects.toThrow('safe integer') diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 5a67a88fe..6f0a36bc4 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -234,6 +234,21 @@ export class KnexMigrations implements MigrationSource { }) }, async down (knex) { + // MySQL may discard the automatically-created userId index once one + // of these wider indexes can support the foreign key. Recreate the + // original index before removing both wider indexes. + if (await determineDBType(knex) === 'MySQL') { + const result = await knex.raw( + 'SHOW INDEX FROM ?? WHERE Key_name = ?', + ['outputs', 'outputs_userid_foreign'] + ) + const indexes = result[0] as unknown[] + if (indexes.length === 0) { + await knex.schema.alterTable('outputs', table => { + table.index(['userId'], 'outputs_userid_foreign') + }) + } + } await knex.schema.alterTable('tx_labels_map', table => { table.dropIndex(['transactionId', 'isDeleted'], 'idx_tx_labels_map_tx_deleted') }) @@ -255,6 +270,22 @@ export class KnexMigrations implements MigrationSource { }) }, async down (knex) { + // MySQL may discard the automatically-created index that supports the + // spentBy foreign key after this migration adds an equivalent named + // index. Restore the original support index before removing ours so a + // complete rollback remains possible. + if (await determineDBType(knex) === 'MySQL') { + const result = await knex.raw( + 'SHOW INDEX FROM ?? WHERE Key_name = ?', + ['outputs', 'outputs_spentby_foreign'] + ) + const indexes = result[0] as unknown[] + if (indexes.length === 0) { + await knex.schema.alterTable('outputs', table => { + table.index(['spentBy'], 'outputs_spentby_foreign') + }) + } + } await knex.schema.alterTable('outputs', table => { table.dropIndex(['spentBy'], 'idx_outputs_spentby') table.dropIndex(['userId', 'basketId', 'spendable', 'satoshis'], 'idx_outputs_user_basket_spendable_satoshis') diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTx.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTx.ts index 40a3bd8c4..1d589843f 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTx.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTx.ts @@ -87,12 +87,17 @@ export class EntityProvenTx extends EntityBase { /** * @returns desirialized `MerklePath` object, value is cached. */ - getMerklePath(): MerklePath { - this._mp ??= MerklePath.fromBinary(this.api.merklePath) - return this._mp + getMerklePath(validateRoots: boolean = true): MerklePath { + if (validateRoots) { + this._mp ??= MerklePath.fromBinary(this.api.merklePath) + return this._mp + } + this._mpUnchecked ??= MerklePath.fromBinary(this.api.merklePath, true, false) + return this._mpUnchecked } _mp?: MerklePath + _mpUnchecked?: MerklePath get provenTxId() { return this.api.provenTxId diff --git a/packages/wallet/wallet-toolbox/src/storage/storageProviderHelpers.ts b/packages/wallet/wallet-toolbox/src/storage/storageProviderHelpers.ts index a3f93b095..f5d05aea3 100644 --- a/packages/wallet/wallet-toolbox/src/storage/storageProviderHelpers.ts +++ b/packages/wallet/wallet-toolbox/src/storage/storageProviderHelpers.ts @@ -67,8 +67,14 @@ export async function mergeInputBeefs ( for (const input of tx.inputs) { const sourceTXID = input.sourceTXID ?? '' if (sourceTXID === '') throw new WERR_INTERNAL('req all transaction inputs must have valid sourceTXID') - if (beef.findTxid(sourceTXID) != null) continue - if ((requiredLevels == null || requiredLevels === 0) && (knownTxids?.includes(sourceTXID) === true)) { + const existing = beef.findTxid(sourceTXID) + const callerKnows = (requiredLevels == null || requiredLevels === 0) && + (knownTxids?.includes(sourceTXID) === true) + // Stored input BEEFs can contain txid-only placeholders supplied by a + // different client. They are not proof for the current caller unless that + // caller also declared the txid known (or explicitly trusts this storage). + if (existing != null && (!existing.isTxidOnly || callerKnows || trustSelf === 'known')) continue + if (callerKnows) { beef.mergeTxidOnly(sourceTXID) } else { await getValidBeef(sourceTXID, beef, trustSelf, knownTxids, trx, requiredLevels) diff --git a/packages/wallet/wallet-toolbox/src/utility/ScriptTemplateBRC29.ts b/packages/wallet/wallet-toolbox/src/utility/ScriptTemplateBRC29.ts index a82fdbc32..75d89f323 100644 --- a/packages/wallet/wallet-toolbox/src/utility/ScriptTemplateBRC29.ts +++ b/packages/wallet/wallet-toolbox/src/utility/ScriptTemplateBRC29.ts @@ -7,10 +7,11 @@ import { LockingScript, P2PKH, PrivateKey, + PublicKey, Script, CachedKeyDeriver } from '@bsv/sdk' -import { asBsvSdkPrivateKey, verifyTruthy } from './utilityHelpers' +import { verifyTruthy } from './utilityHelpers' export const brc29ProtocolID: WalletProtocol = [2, '3241645161d8'] @@ -39,7 +40,13 @@ export class ScriptTemplateBRC29 implements ScriptTemplate { } getKeyDeriver (privKey: PrivateKey | HexString): KeyDeriverApi { - if (typeof privKey === 'string') privKey = PrivateKey.fromHex(privKey) + if (this.params.keyDeriver?.rootKey === privKey) return this.params.keyDeriver + if (typeof privKey === 'string') { + // The wallet signing path supplies its existing root key on every input. + // Avoid parsing the same hex key into a BigNumber for each template. + if (this.params.keyDeriver?.rootKey.toHex() === privKey) return this.params.keyDeriver + privKey = PrivateKey.fromHex(privKey) + } if (this.params.keyDeriver == null || this.params.keyDeriver.rootKey.toHex() !== privKey.toHex()) { return new CachedKeyDeriver(privKey) } return this.params.keyDeriver } @@ -53,16 +60,22 @@ export class ScriptTemplateBRC29 implements ScriptTemplate { } unlock ( - unlockerPrivKey: string, - lockerPubKey: string, + unlockerPrivKey: PrivateKey | HexString, + lockerPubKey: PublicKey | string, sourceSatoshis?: number, lockingScript?: Script ): ScriptTemplateUnlock { const derivedPrivateKey = this.getKeyDeriver(unlockerPrivKey) .derivePrivateKey(brc29ProtocolID, this.getKeyID(), lockerPubKey) - .toHex() - const r = this.p2pkh.unlock(asBsvSdkPrivateKey(derivedPrivateKey), 'all', false, sourceSatoshis, lockingScript) - return r + return this.unlockWithDerivedPrivateKey(derivedPrivateKey, sourceSatoshis, lockingScript) + } + + unlockWithDerivedPrivateKey ( + derivedPrivateKey: PrivateKey, + sourceSatoshis?: number, + lockingScript?: Script + ): ScriptTemplateUnlock { + return this.p2pkh.unlock(derivedPrivateKey, 'all', false, sourceSatoshis, lockingScript) } /** diff --git a/packages/wallet/wallet-toolbox/src/utility/__tests__/ScriptTemplateBRC29.test.ts b/packages/wallet/wallet-toolbox/src/utility/__tests__/ScriptTemplateBRC29.test.ts new file mode 100644 index 000000000..1990205e7 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/__tests__/ScriptTemplateBRC29.test.ts @@ -0,0 +1,63 @@ +import { CachedKeyDeriver, P2PKH, PrivateKey, PublicKey, Script, Transaction } from '@bsv/sdk' +import { brc29ProtocolID, ScriptTemplateBRC29 } from '../ScriptTemplateBRC29' + +describe('ScriptTemplateBRC29', () => { + it('reuses only a matching cached root key for string and object inputs', () => { + const cachedRoot = new PrivateKey(38) + const template = new ScriptTemplateBRC29({ + derivationPrefix: 'cache-prefix', + derivationSuffix: 'cache-suffix', + keyDeriver: new CachedKeyDeriver(cachedRoot) + }) + const differentStringRoot = new PrivateKey(39) + const differentObjectRoot = new PrivateKey(40) + + expect(template.getKeyDeriver(cachedRoot)).toBe(template.params.keyDeriver) + expect(template.getKeyDeriver(differentStringRoot.toHex()).rootKey.toHex()) + .toBe(differentStringRoot.toHex()) + expect(template.getKeyDeriver(differentObjectRoot).rootKey.toHex()) + .toBe(differentObjectRoot.toHex()) + }) + + it('preserves string-key signing bytes when passed cached key objects', async () => { + const rootKey = new PrivateKey(41) + const counterparty = new PrivateKey(42).toPublicKey() + const params = { + derivationPrefix: 'equivalence-prefix', + derivationSuffix: 'equivalence-suffix' + } + const keyID = `${params.derivationPrefix} ${params.derivationSuffix}` + const baselineDeriver = new CachedKeyDeriver(PrivateKey.fromHex(rootKey.toHex())) + const derived = baselineDeriver.derivePrivateKey(brc29ProtocolID, keyID, counterparty.toString()) + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0xffffffff, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 2, lockingScript: new P2PKH().lock(derived.toAddress()) }) + + const sign = async (unlockingScriptTemplate: ReturnType): Promise => { + const tx = new Transaction() + tx.addInput({ sourceTransaction: source, sourceOutputIndex: 0, unlockingScriptTemplate }) + tx.addOutput({ satoshis: 1, lockingScript: Script.fromASM('OP_TRUE') }) + await tx.sign() + return tx.toHex() + } + + const template = new ScriptTemplateBRC29({ ...params, keyDeriver: new CachedKeyDeriver(rootKey) }) + const stringKeyBytes = await sign(template.unlock(rootKey.toHex(), counterparty.toString())) + const objectKeyBytes = await sign(template.unlock(rootKey, PublicKey.fromString(counterparty.toString()))) + const batchDerived = template.params.keyDeriver.derivePrivateKeys?.([{ + protocolID: brc29ProtocolID, + keyID, + counterparty + }])[0] + const batchBytes = await sign(template.unlockWithDerivedPrivateKey(batchDerived!, 2, source.outputs[0].lockingScript)) + const baselineBytes = await sign(new P2PKH().unlock(derived, 'all', false, 2, source.outputs[0].lockingScript)) + + expect(objectKeyBytes).toBe(stringKeyBytes) + expect(objectKeyBytes).toBe(baselineBytes) + expect(batchBytes).toBe(baselineBytes) + }) +}) diff --git a/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts b/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts index 03a0ae0b4..2c696c47f 100644 --- a/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts +++ b/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts @@ -80,6 +80,67 @@ describe('in-memory action batch workspace', () => { await ctx.wallet.destroy() }) + test('default mode stages noSend actions and commits them through sendWith', async () => { + // Recreate the wallet without selecting an action-batch mode to guard the + // default, automatically negotiated lifecycle introduced in #289. + ctx.wallet = new Wallet({ + chain: ctx.chain, + keyDeriver: ctx.keyDeriver, + storage: ctx.storage, + services: ctx.services, + monitor: ctx.monitor + }) + const begin = jest.spyOn(ctx.storage, 'beginActionBatch') + const commit = jest.spyOn(ctx.storage, 'commitActionBatch') + const legacyCreate = jest.spyOn(ctx.storage, 'createAction') + + const staged = await ctx.wallet.createAction(actionArgs()) + + expect(ctx.wallet.actionBatch.mode).toBe('auto') + expect(begin).toHaveBeenCalledTimes(1) + expect(legacyCreate).not.toHaveBeenCalled() + const storedBeforeCommit = await ctx.activeStorage.findTransactions({ + partial: { userId: ctx.userId, txid: staged.txid }, + noRawTx: true + }) + expect(storedBeforeCommit).toHaveLength(0) + await expect(ctx.wallet.listNoSendActions({ labels: ['action batch workload'] })).resolves.toEqual( + expect.objectContaining({ + actions: expect.arrayContaining([expect.objectContaining({ txid: staged.txid, status: 'nosend' })]) + }) + ) + + const sent = await ctx.wallet.createAction({ + description: 'Commit and send the staged noSend action', + options: { sendWith: [staged.txid!] } + }) + expect(sent.sendWithResults).toContainEqual(expect.objectContaining({ txid: staged.txid })) + expect(commit).toHaveBeenCalledTimes(1) + const storedAfterCommit = await ctx.activeStorage.findTransactions({ + partial: { userId: ctx.userId, txid: staged.txid }, + noRawTx: true + }) + expect(storedAfterCommit).toHaveLength(1) + const begun = await begin.mock.results[0].value + expect((await ctx.activeStorage.findActionBatch(ctx.userId, begun.batchId))?.status).toBe('committed') + }) + + test('staged noSend actions listed by txid can be aborted without exposing an internal reference', async () => { + const begin = jest.spyOn(ctx.storage, 'beginActionBatch') + const created = await ctx.wallet.createAction(actionArgs()) + const listed = await ctx.wallet.listNoSendActions({ labels: ['action batch workload'] }) + expect(listed.actions).toContainEqual(expect.objectContaining({ txid: created.txid, status: 'nosend' })) + + await expect(ctx.wallet.abortAction({ reference: created.txid! })).resolves.toEqual({ aborted: true }) + const stored = await ctx.activeStorage.findTransactions({ + partial: { userId: ctx.userId, txid: created.txid }, + noRawTx: true + }) + expect(stored).toHaveLength(0) + const begun = await begin.mock.results[0].value + expect((await ctx.activeStorage.findActionBatch(ctx.userId, begun.batchId))?.status).toBe('aborted') + }) + test('dependent chain uses one begin, no middle storage calls, and one commit', async () => { const begin = jest.spyOn(ctx.storage, 'beginActionBatch') const extend = jest.spyOn(ctx.storage, 'extendActionBatch') @@ -340,8 +401,9 @@ describe('in-memory action batch workspace', () => { expect(commit).toHaveBeenCalledTimes(1) expect(commit.mock.calls[0][0].actions[0].plan.inputs.every(input => input.sourceTransaction == null)).toBe(true) expect(events.some(event => event.name === 'wallet.create_action' && event.spanStatus === 'ok')).toBe(true) - expect(events.some(event => event.name === 'wallet.create_action.prepare_known_txids' && event.spanStatus === 'ok')) - .toBe(true) + expect( + events.some(event => event.name === 'wallet.create_action.prepare_known_txids' && event.spanStatus === 'ok') + ).toBe(true) expect(events.some(event => event.name === 'wallet.sign_action' && event.spanStatus === 'ok')).toBe(true) expect(events.some(event => event.name === 'wallet.crypto.transaction_sign')).toBe(true) }) diff --git a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts index b6befe56c..94abc9a43 100644 --- a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts @@ -21,7 +21,7 @@ describe('KnexMigrations tests', () => { knexs.push(knexSQLite) if (env.runMySQL) { - const knexMySQL = _tu.createLocalMySQL('migratetest') + const knexMySQL = _tu.createLocalMySQL(process.env.MYSQL_MIGRATION_TEST_DATABASE ?? 'migratetest') knexs.push(knexMySQL) } }) @@ -48,13 +48,11 @@ describe('KnexMigrations tests', () => { } const count = Object.keys(config.migrationSource.migrations).length for (let i = 0; i < count; i++) { - try { - const r = await knex.migrate.down(config) - expect(r).toBeTruthy() - } catch { - break - } + if (await knex.migrate.currentVersion(config) === 'none') break + const r = await knex.migrate.down(config) + expect(r).toBeTruthy() } + expect(await knex.migrate.currentVersion(config)).toBe('none') } done0 = true }) @@ -206,4 +204,93 @@ describe('KnexMigrations tests', () => { await knex.destroy() } }) + + test.each([ + { + migrationName: '2026-02-27-001 add listOutputs path indexes', + supportIndex: 'outputs_userid_foreign', + addedIndexes: [ + 'idx_tx_labels_map_tx_deleted', + 'idx_output_tags_map_output_deleted_tag', + 'idx_outputs_user_basket_spendable_outputid', + 'idx_outputs_user_spendable_outputid' + ] + }, + { + migrationName: '2026-02-27-002 add createAction path indexes', + supportIndex: 'outputs_spentby_foreign', + addedIndexes: [ + 'idx_outputs_spentby', + 'idx_outputs_user_basket_spendable_satoshis' + ] + } + ])('6 restores the MySQL support index before rolling back $migrationName', async ({ + migrationName, + supportIndex, + addedIndexes + }) => { + const index = jest.fn() + const dropIndex = jest.fn() + const table = { index, dropIndex } + const raw = jest.fn() + .mockResolvedValueOnce([[{ database_type: 'MySQL' }]]) + .mockResolvedValueOnce([[]]) + const alterTable = jest.fn(async ( + _tableName: string, + callback: (tableBuilder: typeof table) => void + ) => { callback(table) }) + const knex = { raw, schema: { alterTable } } as unknown as Knex + const source = new KnexMigrations('test', 'MySQL rollback test', '1'.repeat(64), 1000) + const migration = await source.getMigration(migrationName) + + await migration.down?.(knex) + + expect(index).toHaveBeenCalledWith(expect.any(Array), supportIndex) + expect(dropIndex.mock.calls.map(call => call[1])).toEqual(addedIndexes) + }) + + test.each([ + '2026-02-27-001 add listOutputs path indexes', + '2026-02-27-002 add createAction path indexes' + ])('7 preserves an existing MySQL foreign-key support index while rolling back %s', async migrationName => { + const index = jest.fn() + const dropIndex = jest.fn() + const table = { index, dropIndex } + const raw = jest.fn() + .mockResolvedValueOnce([[{ database_type: 'MySQL' }]]) + .mockResolvedValueOnce([[{ Key_name: 'existing_support_index' }]]) + const alterTable = jest.fn(async ( + _tableName: string, + callback: (tableBuilder: typeof table) => void + ) => { callback(table) }) + const knex = { raw, schema: { alterTable } } as unknown as Knex + const source = new KnexMigrations('test', 'MySQL rollback test', '1'.repeat(64), 1000) + const migration = await source.getMigration(migrationName) + + await migration.down?.(knex) + + expect(index).not.toHaveBeenCalled() + expect(dropIndex).toHaveBeenCalled() + }) + + test.each([ + '2026-02-27-001 add listOutputs path indexes', + '2026-02-27-002 add createAction path indexes' + ])('8 rolls back %s without MySQL support-index repair on SQLite', async migrationName => { + const dropIndex = jest.fn() + const raw = jest.fn(async () => await Promise.reject( + Object.assign(new Error('SQLite does not implement VERSION()'), { code: 'SQLITE_ERROR' }) + )) + const alterTable = jest.fn(async ( + _tableName: string, + callback: (tableBuilder: { dropIndex: typeof dropIndex }) => void + ) => { callback({ dropIndex }) }) + const knex = { raw, schema: { alterTable } } as unknown as Knex + const source = new KnexMigrations('test', 'SQLite rollback test', '1'.repeat(64), 1000) + const migration = await source.getMigration(migrationName) + + await migration.down?.(knex) + + expect(dropIndex).toHaveBeenCalled() + }) }) diff --git a/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts b/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts index a5cb1c435..75a96d97e 100644 --- a/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts @@ -1,5 +1,5 @@ -import { Beef } from '@bsv/sdk' -import { shareReqsWithWorld } from '../../src/storage/methods/processAction' +import { Beef, Telemetry, TelemetryEvent } from '@bsv/sdk' +import { processAction, shareReqsWithWorld } from '../../src/storage/methods/processAction' import { TableProvenTxReq } from '../../src/storage/schema/tables/TableProvenTxReq' function makeReadyReq (): TableProvenTxReq { @@ -34,6 +34,47 @@ function makeStorageFake () { } describe('processAction shareReqsWithWorld', () => { + test('reports the successful processAction and share spans', async () => { + const events: TelemetryEvent[] = [] + const storage = { + ...makeStorageFake(), + telemetry: new Telemetry({ sink: { capture: event => events.push(event) } }) + } + + const result = await processAction(storage as any, { userId: 1 }, { + isNewTx: false, + isSendWith: false, + isNoSend: true, + isDelayed: false, + sendWith: [] + }) + + expect(result.sendWithResults).toEqual([]) + expect(events).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: 'wallet.storage.process_action.share', spanStatus: 'ok' }), + expect.objectContaining({ + name: 'wallet.storage.process_action', + spanStatus: 'ok', + attributes: expect.objectContaining({ 'action.send_result_count': 0 }) + }) + ])) + }) + + test('preserves the non-instrumented processAction path when telemetry is disabled', async () => { + const storage = { + ...makeStorageFake(), + telemetry: new Telemetry() + } + + await expect(processAction(storage as any, { userId: 1 }, { + isNewTx: false, + isSendWith: false, + isNoSend: true, + isDelayed: false, + sendWith: [] + })).resolves.toMatchObject({ sendWithResults: [] }) + }) + test('delayed sends do not build aggregate BEEF before scheduling', async () => { const req = makeReadyReq() const storage = { diff --git a/packages/wallet/wallet-toolbox/test/utils/TestUtilsWalletStorage.ts b/packages/wallet/wallet-toolbox/test/utils/TestUtilsWalletStorage.ts index 08f0d1bbd..1d1533400 100644 --- a/packages/wallet/wallet-toolbox/test/utils/TestUtilsWalletStorage.ts +++ b/packages/wallet/wallet-toolbox/test/utils/TestUtilsWalletStorage.ts @@ -17,6 +17,8 @@ import { SignActionArgs, SignActionResult, Transaction, + type TelemetryConfig, + type SpendVerifierInterface, Utils, WalletAction, WalletActionInput, @@ -58,6 +60,7 @@ import { TableMonitorEvent } from '../../src/storage/schema/tables/TableMonitorE import { TableCommission } from '../../src/storage/schema/tables/TableCommission' import { asArray } from '../../src/utility/utilityHelpers.noBuffer' import { ScriptTemplateBRC29 } from '../../src/utility/ScriptTemplateBRC29' +import type { ActionBatchMode } from '../../src/signer/actionBatch/ActionBatchWorkspace' dotenv.config() @@ -307,6 +310,9 @@ export abstract class TestUtilsWalletStorage { active?: WalletStorageProvider backups?: WalletStorageProvider[] privKeyHex?: string + actionBatchMode?: ActionBatchMode + telemetry?: TelemetryConfig + scriptVerifier?: SpendVerifierInterface }): Promise { args.chain ||= 'test' args.rootKeyHex ||= '1'.repeat(64) @@ -337,7 +343,10 @@ export abstract class TestUtilsWalletStorage { storage, services, monitor, - privilegedKeyManager + privilegedKeyManager, + actionBatchMode: args.actionBatchMode, + telemetry: args.telemetry, + scriptVerifier: args.scriptVerifier }) const r: TestWalletOnly = { rootKey, @@ -400,16 +409,22 @@ export abstract class TestUtilsWalletStorage { rootKeyHex?: string endpointUrl?: string chain?: Chain + actionBatchMode?: ActionBatchMode + telemetry?: TelemetryConfig + scriptVerifier?: SpendVerifierInterface }): Promise { args.chain ||= 'test' const wo = await _tu.createWalletOnly({ chain: args.chain, - rootKeyHex: args.rootKeyHex + rootKeyHex: args.rootKeyHex, + actionBatchMode: args.actionBatchMode, + telemetry: args.telemetry, + scriptVerifier: args.scriptVerifier }) args.endpointUrl ||= args.chain === 'main' ? 'https://storage.babbage.systems' : 'https://staging-storage.babbage.systems' - const client = new StorageClient(wo.wallet, args.endpointUrl) + const client = new StorageClient(wo.wallet, args.endpointUrl, { telemetry: args.telemetry }) await wo.storage.addWalletStorageProvider(client) return wo } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e03322f03..1f453576f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,7 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: - brace-expansion@<=5.0.7: 5.0.8 + brace-expansion@<5.0.9: 5.0.9 typed-rest-client@2.3.1>qs: 6.15.3 importers: @@ -1601,6 +1601,9 @@ importers: '@bsv/sdk': specifier: workspace:^ version: link:../../sdk + '@bsv/verifast': + specifier: workspace:^ + version: link:../../verifast '@jest/globals': specifier: ^30.4.1 version: 30.4.1 @@ -4709,8 +4712,8 @@ packages: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} - brace-expansion@5.0.8: - resolution: {integrity: sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==} + brace-expansion@5.0.9: + resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==} engines: {node: 20 || >=22} braces@3.0.3: @@ -5542,8 +5545,8 @@ packages: fast-string-width@3.0.2: resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} - fast-uri@3.1.4: - resolution: {integrity: sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==} + fast-uri@3.1.5: + resolution: {integrity: sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==} fast-wrap-ansi@0.2.2: resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==} @@ -5907,8 +5910,8 @@ packages: invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} - ip-address@10.2.0: - resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} + ip-address@10.3.1: + resolution: {integrity: sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==} engines: {node: '>= 12'} ipaddr.js@1.9.1: @@ -11521,14 +11524,14 @@ snapshots: ajv@8.18.0: dependencies: fast-deep-equal: 3.1.3 - fast-uri: 3.1.4 + fast-uri: 3.1.5 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 - fast-uri: 3.1.4 + fast-uri: 3.1.5 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 @@ -11746,7 +11749,7 @@ snapshots: transitivePeerDependencies: - supports-color - brace-expansion@5.0.8: + brace-expansion@5.0.9: dependencies: balanced-match: 4.0.4 @@ -12534,7 +12537,7 @@ snapshots: dependencies: debug: 4.4.3 express: 5.2.1 - ip-address: 10.2.0 + ip-address: 10.3.1 transitivePeerDependencies: - supports-color @@ -12597,7 +12600,7 @@ snapshots: dependencies: fast-string-truncated-width: 3.0.3 - fast-uri@3.1.4: {} + fast-uri@3.1.5: {} fast-wrap-ansi@0.2.2: dependencies: @@ -13010,7 +13013,7 @@ snapshots: dependencies: loose-envify: 1.4.0 - ip-address@10.2.0: {} + ip-address@10.3.1: {} ipaddr.js@1.9.1: {} @@ -14492,15 +14495,15 @@ snapshots: minimatch@10.2.5: dependencies: - brace-expansion: 5.0.8 + brace-expansion: 5.0.9 minimatch@3.1.5: dependencies: - brace-expansion: 5.0.8 + brace-expansion: 5.0.9 minimatch@9.0.9: dependencies: - brace-expansion: 5.0.8 + brace-expansion: 5.0.9 minimist@1.2.8: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 144c598a3..9508cd09c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -33,10 +33,10 @@ trustPolicy: no-downgrade trustPolicyIgnoreAfter: 525600 # Jest 30's current coverage/reporting graph still constrains minimatch to -# brace-expansion 1.x/2.x. GHSA-mh99-v99m-4gvg is only fixed in 5.0.8, so keep -# this narrow, tested substitution until Jest adopts minimatch 10.2.5 or newer. +# brace-expansion 1.x/2.x. Keep this narrow, tested substitution on the latest +# security-patched release until Jest adopts it directly. overrides: - brace-expansion@<=5.0.7: 5.0.8 + brace-expansion@<5.0.9: 5.0.9 # Stryker's latest typed-rest-client 2.3.1 pins vulnerable qs 6.15.1 # exactly. Keep this parent-scoped substitution until upstream publishes a # release that depends on qs 6.15.2 or newer. diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index 6b6fd4dfa..e43cf28d7 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -31,12 +31,12 @@ test('pnpm override parsing preserves scoped parent selectors', () => { parsePnpmOverrides(` minimumReleaseAge: 1440 overrides: - brace-expansion@<=5.0.7: 5.0.8 + brace-expansion@<5.0.9: 5.0.9 'typed-rest-client@2.3.1>qs': 6.15.3 trustPolicy: no-downgrade `), [ - { selector: 'brace-expansion@<=5.0.7', value: '5.0.8' }, + { selector: 'brace-expansion@<5.0.9', value: '5.0.9' }, { selector: 'typed-rest-client@2.3.1>qs', value: '6.15.3' } ] ) diff --git a/scripts/patch-coverage.mjs b/scripts/patch-coverage.mjs index 695ae3f07..d3d820f27 100644 --- a/scripts/patch-coverage.mjs +++ b/scripts/patch-coverage.mjs @@ -17,9 +17,10 @@ const EXCLUDED_SOURCE_PATTERNS = [ // LCOV is unsatisfiable: a package that adds or edits jest.config.cjs, // vitest.config.ts or similar could never clear this gate. /(?:^|\/)[^/]*\.config\.[cm]?[jt]s$/, - // Benchmark orchestration and type-only IndexedDB schema declarations have - // no executable statements for Jest/Istanbul to instrument. + // Benchmark orchestration and type-only declarations have no executable + // statements for Jest/Istanbul to instrument. /packages\/sdk\/scripts\/run-benchmarks\.js$/, + /\.interfaces\.[cm]?[jt]sx?$/, /packages\/wallet\/wallet-toolbox\/src\/storage\/schema\/StorageIdbSchema\.ts$/ ] diff --git a/scripts/patch-coverage.test.mjs b/scripts/patch-coverage.test.mjs index 2afbd0b63..fd963d0a6 100644 --- a/scripts/patch-coverage.test.mjs +++ b/scripts/patch-coverage.test.mjs @@ -77,7 +77,7 @@ test('patch coverage fails closed when a changed production file is absent from }) }) -test('patch coverage ignores non-instrumented configuration, benchmarks, and type-only schemas', () => { +test('patch coverage ignores non-instrumented configuration, benchmarks, and type-only declarations', () => { const changed = changedLinesFromDiff(`diff --git a/packages/helpers/example/jest.config.cjs b/packages/helpers/example/jest.config.cjs +++ b/packages/helpers/example/jest.config.cjs @@ -94,6 +94,9 @@ diff --git a/packages/sdk/scripts/run-benchmarks.js b/packages/sdk/scripts/run-b diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts b/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts @@ -0,0 +1,12 @@ +diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts ++++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +@@ -0,0 +1,12 @@ diff --git a/packages/helpers/example/src/index.ts b/packages/helpers/example/src/index.ts +++ b/packages/helpers/example/src/index.ts @@ -0,0 +1 @@ diff --git a/scripts/repository-health.test.mjs b/scripts/repository-health.test.mjs index 00ece13dd..a7a0e3b2d 100644 --- a/scripts/repository-health.test.mjs +++ b/scripts/repository-health.test.mjs @@ -62,7 +62,7 @@ test('workspace discovery exactly matches the 38-project registry', () => { }) test('current repository health controls and ratchet are internally consistent', () => { - const result = evaluateRepositoryHealth({ today: '2026-07-30' }) + const result = evaluateRepositoryHealth({ today: '2026-08-04' }) assert.deepEqual(result.errors, []) assert.equal(result.projects.length, 38) diff --git a/tools/codegen/node/package-lock.json b/tools/codegen/node/package-lock.json index 3bdeb3754..b5d711dd6 100644 --- a/tools/codegen/node/package-lock.json +++ b/tools/codegen/node/package-lock.json @@ -7,11 +7,11 @@ "": { "name": "@bsv/ts-stack-codegen", "version": "0.0.0", + "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "openapi-typescript": "7.13.0", "typescript": "5.9.3" - }, - "license": "SEE LICENSE IN LICENSE.txt" + } }, "node_modules/@babel/code-frame": { "version": "7.29.7", @@ -113,9 +113,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" diff --git a/tools/codegen/node/package.json b/tools/codegen/node/package.json index 9848033e5..17715003b 100644 --- a/tools/codegen/node/package.json +++ b/tools/codegen/node/package.json @@ -13,7 +13,7 @@ "minimatch": "10.2.5" }, "minimatch@10.2.5": { - "brace-expansion": "5.0.8" + "brace-expansion": "5.0.9" } }, "license": "SEE LICENSE IN LICENSE.txt"