diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index f5227451b..1f3ac46cf 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -17,3 +17,5 @@ paths-ignore: - infra/chaintracks-server/src/security/edgePolicy.ts - packages/overlays/overlay-express/src/security/edgePolicy.ts - packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts + - infra/uhrp-server-cloud-bucket/src/resourceLimits.ts + - infra/wallet-infra/src/KnexPaymentReplayStore.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67a71cc75..7eeba3ac7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,7 @@ jobs: run: | node scripts/sync-service-rate-limit-policy.mjs --check node scripts/sync-service-edge-policy.mjs --check + node scripts/sync-service-runtime-copies.mjs --check - name: Enforce inventory, exception, baseline, and contract ratchets run: >- @@ -714,8 +715,23 @@ jobs: name: build-outputs path: .ci-artifacts - run: tar --extract --gzip --file .ci-artifacts/build-outputs.tar.gz - - name: Verify packed mobile consumer with Metro and Hermes - run: pnpm --filter @bsv/wallet-toolbox-mobile run test:mobile + - name: Verify packed mobile consumer with Metro and Hermes and retain source coverage + run: | + pnpm --filter @bsv/wallet-toolbox-mobile run test:mobile + pnpm --filter @bsv/wallet-toolbox-mobile run test:coverage + node scripts/normalize-lcov-paths.mjs \ + packages/wallet/wallet-toolbox/mobile/coverage/lcov.info \ + packages/wallet/wallet-toolbox + mkdir -p .coverage-output + cp packages/wallet/wallet-toolbox/mobile/coverage/lcov.info \ + .coverage-output/wallet-mobile.lcov.info + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: coverage-wallet-mobile + path: .coverage-output/ + if-no-files-found: error + include-hidden-files: true + retention-days: 1 coverage-sdk: name: Coverage / SDK @@ -1045,6 +1061,7 @@ jobs: (needs.coverage-did.result == 'success' || needs.coverage-did.result == 'skipped') && (needs.coverage-wallet.result == 'success' || needs.coverage-wallet.result == 'skipped') && (needs.coverage-wallet-monitor.result == 'success' || needs.coverage-wallet-monitor.result == 'skipped') && + (needs.wallet-mobile-platform.result == 'success' || needs.wallet-mobile-platform.result == 'skipped') && (needs.coverage-verifast.result == 'success' || needs.coverage-verifast.result == 'skipped') needs: - prepare @@ -1052,6 +1069,7 @@ jobs: - coverage-did - coverage-wallet - coverage-wallet-monitor + - wallet-mobile-platform - coverage-verifast - coverage-other runs-on: ubuntu-latest diff --git a/.sonarcloud.properties b/.sonarcloud.properties index fe8cab945..6c28ed3e4 100644 --- a/.sonarcloud.properties +++ b/.sonarcloud.properties @@ -10,7 +10,7 @@ sonar.exclusions=packages/verifast/src/wasm/bdk-core.*,conformance/generated/**, # into self-contained Docker/package build contexts and checked byte-for-byte # in CI. Analyze the code for issues, but do not report intentional generated # copies as source duplication. -sonar.cpd.exclusions=**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.man.test.ts,**/__test__/**,**/__tests__/**,**/test/**,**/tests/**,**/*.vectors.ts,**/eslint.config.js,infra/wab/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/rateLimitPolicy.ts,infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts,infra/message-box-server/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts,infra/message-box-server/src/security/edgePolicy.ts,infra/chaintracks-server/src/security/edgePolicy.ts,packages/overlays/overlay-express/src/security/edgePolicy.ts,packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts +sonar.cpd.exclusions=**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.man.test.ts,**/__test__/**,**/__tests__/**,**/test/**,**/tests/**,**/*.vectors.ts,**/eslint.config.js,infra/wab/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/rateLimitPolicy.ts,infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts,infra/message-box-server/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts,infra/message-box-server/src/security/edgePolicy.ts,infra/chaintracks-server/src/security/edgePolicy.ts,packages/overlays/overlay-express/src/security/edgePolicy.ts,packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/resourceLimits.ts,infra/wallet-infra/src/KnexPaymentReplayStore.ts # Narrow compatibility exceptions are registered with owner, evidence, review # dates, and objective removal conditions in repository-health/exceptions.json. sonar.issue.ignore.multicriteria=werrProtocolNames,curveSingletonAlias,curveSingletonReturn,scriptOpcodeDispatch diff --git a/conformance/generated/messaging/types.gen.d.ts b/conformance/generated/messaging/types.gen.d.ts index 83d2948ca..bb0ece684 100644 --- a/conformance/generated/messaging/types.gen.d.ts +++ b/conformance/generated/messaging/types.gen.d.ts @@ -633,6 +633,8 @@ export interface operations { limit?: number; /** @default 0 */ offset?: number; + /** @description Compatibility alias for offset; both must match if supplied together. */ + skip?: number; }; }; }; @@ -649,6 +651,8 @@ export interface operations { messages: components["schemas"]["StoredMessage"][]; limit: number; offset: number; + /** @description Offset for the next page; unchanged when the result is empty. */ + nextOffset: number; hasMore: boolean; }; }; diff --git a/conformance/generated/messaging/types.gen.go b/conformance/generated/messaging/types.gen.go index 405f2bcff..2859b445e 100644 --- a/conformance/generated/messaging/types.gen.go +++ b/conformance/generated/messaging/types.gen.go @@ -543,6 +543,9 @@ type ListMessagesJSONBody struct { // Examples: payment_inbox MessageBox string `json:"messageBox"` Offset *int `json:"offset,omitempty"` + + // Skip Compatibility alias for offset; both must match if supplied together. + Skip *int `json:"skip,omitempty"` } // ListMessages200JSONResponseBodyStatus defines parameters for ListMessages. diff --git a/conformance/runner/reports/report.json b/conformance/runner/reports/report.json index 1ed241458..c2947a3a2 100644 --- a/conformance/runner/reports/report.json +++ b/conformance/runner/reports/report.json @@ -1,7 +1,7 @@ { - "timestamp": "2026-07-24T22:49:48.768Z", - "totalVectors": 6650, - "totalFiles": 74, + "timestamp": "2026-08-06T03:52:21.445Z", + "totalVectors": 6681, + "totalFiles": 75, "parseErrors": 0, "suites": [ { @@ -28499,6 +28499,166 @@ } ] }, + { + "name": "transport/air-gap-optical", + "cases": [ + { + "name": "transport.air-gap-optical.crc32.1", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.length.2", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.length.3", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.length.4", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.5", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.6", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.7", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.8", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.9", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.10", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.11", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.12", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.13", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.14", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.encode.15", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.decode.16", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.decode.17", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.decode.18", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.session.19", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.session.20", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.session.21", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.session.22", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.23", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.24", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.25", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.26", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.27", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.28", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.29", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.30", + "pass": true, + "error": null + }, + { + "name": "transport.air-gap-optical.reject.31", + "pass": true, + "error": null + } + ] + }, { "name": "wallet/brc100/abortaction", "cases": [ @@ -33625,4 +33785,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/conformance/runner/reports/results.xml b/conformance/runner/reports/results.xml index 0203cf380..ac9bd1496 100644 --- a/conformance/runner/reports/results.xml +++ b/conformance/runner/reports/results.xml @@ -1,5 +1,5 @@ - + @@ -5743,6 +5743,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs-site/public/assets/asyncapi/authsocket/index.html b/docs-site/public/assets/asyncapi/authsocket/index.html index e348268bd..a27595cda 100644 --- a/docs-site/public/assets/asyncapi/authsocket/index.html +++ b/docs-site/public/assets/asyncapi/authsocket/index.html @@ -1 +1 @@ -AuthSocket WebSocket Protocol · AsyncAPI
AsyncAPI 3.1.0 · specs/messaging/authsocket-asyncapi.yaml

AuthSocket WebSocket Protocol

v1.0.0
AsyncAPI 3.0 specification for the `AuthSocketServer` / `AuthSocket` WebSocket channel used by the BSV MessageBox Server. ## Transport layer `AuthSocketServer` wraps Socket.IO and sits on top of an HTTP server. All Socket.IO events are standard Socket.IO framing; this spec describes the *application-level* event names and payload shapes that flow over the Socket.IO connection. ## BRC-103 mutual authentication Every Socket.IO connection undergoes BRC-103 (`Peer`) handshake before application events are exchanged. The handshake is carried on the **`authMessage`** event using the `AuthMessage` envelope defined in the `@bsv/sdk` `Transport` interface. Once the handshake succeeds the peer's `identityKey` (compressed secp256k1 public key, 66-char hex) is known server-side and stored in memory for the lifetime of the connection. ## Application events After authentication the server emits and listens for the events described below. All application payloads are serialized as JSON inside the BRC-103 `general` message (`Peer.toPeer`). The transport layer (`SocketServerTransport`) wraps them in an `{ eventName, data }` envelope before signing. Source of truth: - `packages/messaging/authsocket/src/AuthSocketServer.ts` - `packages/messaging/authsocket/src/SocketServerTransport.ts` - `packages/messaging/message-box-server/src/index.ts`
2Servers16Channels17Operations0Messages18Schemas

Servers

2

production

wss://messagebox.babbage.systems/

Production MessageBox WebSocket endpoint (Socket.IO over WSS).
host
messagebox.babbage.systems
pathname
/
protocol
wss
description
Production MessageBox WebSocket endpoint (Socket.IO over WSS).

local

ws://localhost:{port}/

Local development Socket.IO server.
host
localhost:{port}
pathname
/
protocol
ws
description
Local development Socket.IO server.
variables
1 field
port
2 fields
default
5001
description
HTTP port the MessageBox Server listens on.

Channels

16

authMessage

authMessage

Low-level Socket.IO event used by `SocketServerTransport` to carry BRC-103 `AuthMessage` frames. This event is NOT an application event; it is emitted and received transparently by the `Peer` class from `@bsv/sdk`. Application developers do not interact with this channel directly — they use the typed events below. See `packages/messaging/authsocket/src/SocketServerTransport.ts`.
address
authMessage
description
Low-level Socket.IO event used by `SocketServerTransport` to carry BRC-103 `AuthMessage` frames. This event is NOT an application event; it is emitted and received transparently by the `Peer` class from `@bsv/sdk`. Application developers do not interact with this channel directly — they use the typed events below. See `packages/messaging/authsocket/src/SocketServerTransport.ts`.
messages
1 field
authMessageFrame
3 fields
name
authMessageFrame
summary
BRC-103 auth frame (both directions — client and server).
payload
1 field
$ref
#/components/schemas/AuthMessage

authenticated

authenticated

Fallback authentication event. The client emits this when its identity key was not included in the Socket.IO handshake. The server validates the key, updates its in-memory `authenticatedSockets` map, and emits `authenticationSuccess` or `authenticationFailed` in response.
address
authenticated
description
Fallback authentication event. The client emits this when its identity key was not included in the Socket.IO handshake. The server validates the key, updates its in-memory `authenticatedSockets` map, and emits `authenticationSuccess` or `authenticationFailed` in response.
messages
1 field
authenticateMessage
3 fields
name
authenticateMessage
summary
Client sends its identity key for post-connection auth.
payload
1 field
$ref
#/components/schemas/AuthenticatePayload

authenticationSuccess

authenticationSuccess

Emitted by the server after successful identity key validation.
address
authenticationSuccess
description
Emitted by the server after successful identity key validation.
messages
1 field
authSuccessMessage
2 fields
name
authSuccessMessage
payload
1 field
$ref
#/components/schemas/AuthSuccessPayload

authenticationFailed

authenticationFailed

Emitted by the server when identity key validation fails.
address
authenticationFailed
description
Emitted by the server when identity key validation fails.
messages
1 field
authFailedMessage
2 fields
name
authFailedMessage
payload
1 field
$ref
#/components/schemas/AuthFailedPayload

joinRoom

joinRoom

Client requests to subscribe to a room. Only authenticated sockets may join rooms. The server responds with `joinedRoom` on success or `joinFailed` on error. Room ID convention: `<recipientIdentityKey>-<messageBoxType>`.
address
joinRoom
description
Client requests to subscribe to a room. Only authenticated sockets may join rooms. The server responds with `joinedRoom` on success or `joinFailed` on error. Room ID convention: `<recipientIdentityKey>-<messageBoxType>`.
messages
1 field
joinRoomMessage
3 fields
name
joinRoomMessage
summary
Room ID string to join.
payload
1 field
$ref
#/components/schemas/JoinRoomPayload

joinedRoom

joinedRoom

Server confirms the client has joined the specified room.
address
joinedRoom
description
Server confirms the client has joined the specified room.
messages
1 field
joinedRoomMessage
2 fields
name
joinedRoomMessage
payload
1 field
$ref
#/components/schemas/JoinedRoomPayload

joinFailed

joinFailed

Emitted when `joinRoom` fails (unauthenticated or invalid roomId).
address
joinFailed
description
Emitted when `joinRoom` fails (unauthenticated or invalid roomId).
messages
1 field
joinFailedMessage
2 fields
name
joinFailedMessage
payload
1 field
$ref
#/components/schemas/JoinFailedPayload

leaveRoom

leaveRoom

Client requests to leave a room.
address
leaveRoom
description
Client requests to leave a room.
messages
1 field
leaveRoomMessage
3 fields
name
leaveRoomMessage
summary
Room ID string to leave.
payload
1 field
$ref
#/components/schemas/LeaveRoomPayload

leftRoom

leftRoom

Server confirms the client has left the room.
address
leftRoom
description
Server confirms the client has left the room.
messages
1 field
leftRoomMessage
2 fields
name
leftRoomMessage
payload
1 field
$ref
#/components/schemas/LeftRoomPayload

leaveFailed

leaveFailed

Emitted when `leaveRoom` fails.
address
leaveFailed
description
Emitted when `leaveRoom` fails.
messages
1 field
leaveFailedMessage
2 fields
name
leaveFailedMessage
payload
1 field
$ref
#/components/schemas/LeaveFailedPayload

sendMessage

sendMessage

Client sends a message to a recipient via WebSocket. The server: 1. Validates the sender is authenticated. 2. Validates `roomId` and `message`. 3. Creates the message box if it does not exist. 4. Inserts the message into the database (with ON CONFLICT IGNORE dedup). 5. Emits `sendMessageAck-{roomId}` back to the sender. 6. Broadcasts `sendMessage-{roomId}` to all connections in the room.
address
sendMessage
description
Client sends a message to a recipient via WebSocket. The server: 1. Validates the sender is authenticated. 2. Validates `roomId` and `message`. 3. Creates the message box if it does not exist. 4. Inserts the message into the database (with ON CONFLICT IGNORE dedup). 5. Emits `sendMessageAck-{roomId}` back to the sender. 6. Broadcasts `sendMessage-{roomId}` to all connections in the room.
messages
1 field
sendMessageMessage
2 fields
name
sendMessageMessage
payload
1 field
$ref
#/components/schemas/WsSendMessagePayload

sendMessageAck

sendMessageAck-{roomId}

Per-room acknowledgement emitted to the sender only after the message is stored. The event name is `sendMessageAck-<roomId>` where `roomId` matches the value in the originating `sendMessage` payload.
address
sendMessageAck-{roomId}
description
Per-room acknowledgement emitted to the sender only after the message is stored. The event name is `sendMessageAck-<roomId>` where `roomId` matches the value in the originating `sendMessage` payload.
parameters
1 field
roomId
1 field
description
The room ID from the originating sendMessage request.
messages
1 field
sendMessageAckMessage
2 fields
name
sendMessageAckMessage
payload
1 field
$ref
#/components/schemas/WsSendMessageAckPayload

sendMessageBroadcast

sendMessage-{roomId}

Broadcast emitted to all connections subscribed to `roomId` after a successful `sendMessage`. The event name is `sendMessage-<roomId>`.
address
sendMessage-{roomId}
description
Broadcast emitted to all connections subscribed to `roomId` after a successful `sendMessage`. The event name is `sendMessage-<roomId>`.
parameters
1 field
roomId
1 field
description
The target room ID.
messages
1 field
sendMessageBroadcastMessage
2 fields
name
sendMessageBroadcastMessage
payload
1 field
$ref
#/components/schemas/WsSendMessageBroadcastPayload

messageFailed

messageFailed

Emitted to the sender when `sendMessage` processing fails.
address
messageFailed
description
Emitted to the sender when `sendMessage` processing fails.
messages
1 field
messageFailedMessage
2 fields
name
messageFailedMessage
payload
1 field
$ref
#/components/schemas/MessageFailedPayload

paymentFailed

paymentFailed

Emitted to unauthenticated sockets that attempt to send a message (same event name reused from older payment-gate logic).
address
paymentFailed
description
Emitted to unauthenticated sockets that attempt to send a message (same event name reused from older payment-gate logic).
messages
1 field
paymentFailedMessage
2 fields
name
paymentFailedMessage
payload
1 field
$ref
#/components/schemas/PaymentFailedPayload

disconnect

disconnect

Standard Socket.IO disconnect event. The server removes the socket from the `authenticatedSockets` map and the `peers` map in `AuthSocketServer`.
address
disconnect
description
Standard Socket.IO disconnect event. The server removes the socket from the `authenticatedSockets` map and the `peers` map in `AuthSocketServer`.
messages
1 field
disconnectMessage
2 fields
name
disconnectMessage
payload
1 field
$ref
#/components/schemas/DisconnectPayload

Operations

17

receiveAuthMessage

receive
Server receives an AuthMessage frame from the client during BRC-103 handshake.
action
receive
channel
1 field
$ref
#/channels/authMessage
summary
Server receives an AuthMessage frame from the client during BRC-103 handshake.
messages
1 item
  1. $ref
    #/channels/authMessage/messages/authMessageFrame

sendAuthMessage

send
Server sends an AuthMessage frame to the client during BRC-103 handshake.
action
send
channel
1 field
$ref
#/channels/authMessage
summary
Server sends an AuthMessage frame to the client during BRC-103 handshake.
messages
1 item
  1. $ref
    #/channels/authMessage/messages/authMessageFrame

receiveAuthenticated

receive
Server receives the client's identity key on the 'authenticated' event.
action
receive
channel
1 field
$ref
#/channels/authenticated
summary
Server receives the client's identity key on the 'authenticated' event.
messages
1 item
  1. $ref
    #/channels/authenticated/messages/authenticateMessage

sendAuthenticationSuccess

send
Server confirms successful identity key validation.
action
send
channel
1 field
$ref
#/channels/authenticationSuccess
summary
Server confirms successful identity key validation.
messages
1 item
  1. $ref
    #/channels/authenticationSuccess/messages/authSuccessMessage

sendAuthenticationFailed

send
Server rejects an invalid identity key.
action
send
channel
1 field
$ref
#/channels/authenticationFailed
summary
Server rejects an invalid identity key.
messages
1 item
  1. $ref
    #/channels/authenticationFailed/messages/authFailedMessage

receiveJoinRoom

receive
Server receives a room join request.
action
receive
channel
1 field
$ref
#/channels/joinRoom
summary
Server receives a room join request.
messages
1 item
  1. $ref
    #/channels/joinRoom/messages/joinRoomMessage

sendJoinedRoom

send
Server confirms room join.
action
send
channel
1 field
$ref
#/channels/joinedRoom
summary
Server confirms room join.
messages
1 item
  1. $ref
    #/channels/joinedRoom/messages/joinedRoomMessage

sendJoinFailed

send
Server signals room join failure.
action
send
channel
1 field
$ref
#/channels/joinFailed
summary
Server signals room join failure.
messages
1 item
  1. $ref
    #/channels/joinFailed/messages/joinFailedMessage

receiveLeaveRoom

receive
Server receives a room leave request.
action
receive
channel
1 field
$ref
#/channels/leaveRoom
summary
Server receives a room leave request.
messages
1 item
  1. $ref
    #/channels/leaveRoom/messages/leaveRoomMessage

sendLeftRoom

send
Server confirms room leave.
action
send
channel
1 field
$ref
#/channels/leftRoom
summary
Server confirms room leave.
messages
1 item
  1. $ref
    #/channels/leftRoom/messages/leftRoomMessage

sendLeaveFailed

send
Server signals room leave failure.
action
send
channel
1 field
$ref
#/channels/leaveFailed
summary
Server signals room leave failure.
messages
1 item
  1. $ref
    #/channels/leaveFailed/messages/leaveFailedMessage

receiveSendMessage

receive
Server receives a message from the client to deliver to a recipient.
action
receive
channel
1 field
$ref
#/channels/sendMessage
summary
Server receives a message from the client to deliver to a recipient.
messages
1 item
  1. $ref
    #/channels/sendMessage/messages/sendMessageMessage

sendSendMessageAck

send
Server acknowledges delivery of a message to the sender.
action
send
channel
1 field
$ref
#/channels/sendMessageAck
summary
Server acknowledges delivery of a message to the sender.
messages
1 item
  1. $ref
    #/channels/sendMessageAck/messages/sendMessageAckMessage

sendSendMessageBroadcast

send
Server broadcasts a new message to all room subscribers.
action
send
channel
1 field
$ref
#/channels/sendMessageBroadcast
summary
Server broadcasts a new message to all room subscribers.
messages
1 item
  1. $ref
    #/channels/sendMessageBroadcast/messages/sendMessageBroadcastMessage

sendMessageFailed

send
Server signals message delivery failure.
action
send
channel
1 field
$ref
#/channels/messageFailed
summary
Server signals message delivery failure.
messages
1 item
  1. $ref
    #/channels/messageFailed/messages/messageFailedMessage

sendPaymentFailed

send
Server signals auth/payment gate rejection.
action
send
channel
1 field
$ref
#/channels/paymentFailed
summary
Server signals auth/payment gate rejection.
messages
1 item
  1. $ref
    #/channels/paymentFailed/messages/paymentFailedMessage

receiveDisconnect

receive
Client disconnects; server cleans up in-memory state.
action
receive
channel
1 field
$ref
#/channels/disconnect
summary
Client disconnects; server cleans up in-memory state.
messages
1 item
  1. $ref
    #/channels/disconnect/messages/disconnectMessage

Schemas

18

PubKeyHex

Compressed secp256k1 public key, 66 hex characters.
type
string
pattern
^0[23][0-9a-fA-F]{64}$
description
Compressed secp256k1 public key, 66 hex characters.

AuthMessage

BRC-103 auth envelope as defined in `@bsv/sdk`. Carried on the low-level `authMessage` Socket.IO event. Not an application-level event.
type
object
description
BRC-103 auth envelope as defined in `@bsv/sdk`. Carried on the low-level `authMessage` Socket.IO event. Not an application-level event.
required
3 items
  1. messageType
  2. version
  3. identityKey
properties
9 fields
messageType
3 fields
type
string
enum
3 items
  1. initialRequest
  2. initialResponse
  3. general
description
- `initialRequest` — first handshake message from initiating peer - `initialResponse` — server's challenge response (includes nonce, signature) - `general` — signed application payload after handshake
version
2 fields
type
string
description
Auth protocol version string.
identityKey
1 field
$ref
#/components/schemas/PubKeyHex
nonce
2 fields
type
string
description
Fresh random nonce (base64) generated by the sender.
yourNonce
2 fields
type
string
description
Echo of the peer's nonce from the previous message.
initialNonce
2 fields
type
string
description
Present in `initialRequest`; absent in subsequent messages.
payload
3 fields
type
array
items
1 field
type
integer
description
Signed application payload (byte array). Empty for handshake messages.
signature
3 fields
type
array
items
1 field
type
integer
description
DER-encoded ECDSA signature over the payload.
requestedCertificates
3 fields
type
object
description
Optional certificate request set (BRC-52 format).
additionalProperties
true

EventEnvelope

Application-level wrapper JSON-encoded inside the BRC-103 `general` message payload. The `SocketServerTransport` encodes/decodes this transparently; application code only sees `eventName` and `data`.
type
object
description
Application-level wrapper JSON-encoded inside the BRC-103 `general` message payload. The `SocketServerTransport` encodes/decodes this transparently; application code only sees `eventName` and `data`.
required
2 items
  1. eventName
  2. data
properties
2 fields
eventName
2 fields
type
string
description
The Socket.IO event name.
data
1 field
description
The event-specific payload.

AuthenticatePayload

Sent by the client on the `authenticated` event when the identity key was not available at connection time (fallback path). The server validates the key and responds with `authenticationSuccess` or `authenticationFailed`.
type
object
description
Sent by the client on the `authenticated` event when the identity key was not available at connection time (fallback path). The server validates the key and responds with `authenticationSuccess` or `authenticationFailed`.
properties
1 field
identityKey
1 field
$ref
#/components/schemas/PubKeyHex

AuthSuccessPayload

type
object
required
1 item
  1. status
properties
1 field
status
2 fields
type
string
enum
1 item
  1. success

AuthFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
2 fields
type
string
description
Human-readable reason for failure.

JoinRoomPayload

The room ID string sent by the client on the `joinRoom` event. Room IDs use the convention `<recipientKey>-<messageBoxType>`. Example: `028d37b9...-payment_inbox`.
type
string
description
The room ID string sent by the client on the `joinRoom` event. Room IDs use the convention `<recipientKey>-<messageBoxType>`. Example: `028d37b9...-payment_inbox`.

JoinedRoomPayload

type
object
required
1 item
  1. roomId
properties
1 field
roomId
1 field
type
string

JoinFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

LeaveRoomPayload

The room ID string sent by the client on the `leaveRoom` event.
type
string
description
The room ID string sent by the client on the `leaveRoom` event.

LeftRoomPayload

type
object
required
1 item
  1. roomId
properties
1 field
roomId
1 field
type
string

LeaveFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

WsSendMessagePayload

Payload for the client-to-server `sendMessage` event.
type
object
description
Payload for the client-to-server `sendMessage` event.
required
2 items
  1. roomId
  2. message
properties
2 fields
roomId
2 fields
type
string
description
Target room. Format: `<recipientKey>-<messageBoxType>`. The server extracts the `messageBoxType` by splitting on `-` and taking the second part; it uses the authenticated sender key from `authenticatedSockets`.
message
3 fields
type
object
required
3 items
  1. messageId
  2. recipient
  3. body
properties
3 fields
messageId
2 fields
type
string
description
Unique identifier for this message (deduplication key).
recipient
1 field
$ref
#/components/schemas/PubKeyHex
body
2 fields
type
string
description
Message body string.

WsSendMessageAckPayload

Acknowledgement emitted by the server on `sendMessageAck-{roomId}` after a successful `sendMessage`. Note: the event name is dynamic and includes the room ID used in the originating request.
type
object
description
Acknowledgement emitted by the server on `sendMessageAck-{roomId}` after a successful `sendMessage`. Note: the event name is dynamic and includes the room ID used in the originating request.
required
2 items
  1. status
  2. messageId
properties
2 fields
status
2 fields
type
string
enum
1 item
  1. success
messageId
1 field
type
string

WsSendMessageBroadcastPayload

Broadcast emitted by the server on `sendMessage-{roomId}` to all connections in the room (including the sender). Note: the event name is dynamic.
type
object
description
Broadcast emitted by the server on `sendMessage-{roomId}` to all connections in the room (including the sender). Note: the event name is dynamic.
required
3 items
  1. sender
  2. messageId
  3. body
properties
3 fields
sender
1 field
$ref
#/components/schemas/PubKeyHex
messageId
1 field
type
string
body
1 field
type
string

MessageFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

PaymentFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

DisconnectPayload

The Socket.IO disconnect reason string (e.g. `transport close`, `server namespace disconnect`).
type
string
description
The Socket.IO disconnect reason string (e.g. `transport close`, `server namespace disconnect`).
Raw YAML source
asyncapi: "3.1.0"

info:
  title: AuthSocket WebSocket Protocol
  version: "1.0.0"
  description: |
    AsyncAPI 3.0 specification for the `AuthSocketServer` / `AuthSocket`
    WebSocket channel used by the BSV MessageBox Server.

    ## Transport layer

    `AuthSocketServer` wraps Socket.IO and sits on top of an HTTP server.
    All Socket.IO events are standard Socket.IO framing; this spec describes
    the *application-level* event names and payload shapes that flow over the
    Socket.IO connection.

    ## BRC-103 mutual authentication

    Every Socket.IO connection undergoes BRC-103 (`Peer`) handshake before
    application events are exchanged. The handshake is carried on the
    **`authMessage`** event using the `AuthMessage` envelope defined in the
    `@bsv/sdk` `Transport` interface.

    Once the handshake succeeds the peer's `identityKey` (compressed secp256k1
    public key, 66-char hex) is known server-side and stored in memory for the
    lifetime of the connection.

    ## Application events

    After authentication the server emits and listens for the events described
    below. All application payloads are serialized as JSON inside the
    BRC-103 `general` message (`Peer.toPeer`). The transport layer
    (`SocketServerTransport`) wraps them in an `{ eventName, data }` envelope
    before signing.

    Source of truth:
    - `packages/messaging/authsocket/src/AuthSocketServer.ts`
    - `packages/messaging/authsocket/src/SocketServerTransport.ts`
    - `packages/messaging/message-box-server/src/index.ts`

servers:
  production:
    host: "messagebox.babbage.systems"
    pathname: "/"
    protocol: wss
    description: Production MessageBox WebSocket endpoint (Socket.IO over WSS).
  local:
    host: "localhost:{port}"
    pathname: "/"
    protocol: ws
    description: Local development Socket.IO server.
    variables:
      port:
        default: "5001"
        description: HTTP port the MessageBox Server listens on.

# ---------------------------------------------------------------------------
# Components
# ---------------------------------------------------------------------------
components:
  schemas:
    PubKeyHex:
      type: string
      pattern: "^0[23][0-9a-fA-F]{64}$"
      description: Compressed secp256k1 public key, 66 hex characters.

    AuthMessage:
      type: object
      description: |
        BRC-103 auth envelope as defined in `@bsv/sdk`. Carried on the
        low-level `authMessage` Socket.IO event. Not an application-level event.
      required: [messageType, version, identityKey]
      properties:
        messageType:
          type: string
          enum: [initialRequest, initialResponse, general]
          description: |
            - `initialRequest`  — first handshake message from initiating peer
            - `initialResponse` — server's challenge response (includes nonce, signature)
            - `general`         — signed application payload after handshake
        version:
          type: string
          description: Auth protocol version string.
        identityKey:
          $ref: "#/components/schemas/PubKeyHex"
        nonce:
          type: string
          description: Fresh random nonce (base64) generated by the sender.
        yourNonce:
          type: string
          description: Echo of the peer's nonce from the previous message.
        initialNonce:
          type: string
          description: Present in `initialRequest`; absent in subsequent messages.
        payload:
          type: array
          items:
            type: integer
          description: Signed application payload (byte array). Empty for handshake messages.
        signature:
          type: array
          items:
            type: integer
          description: DER-encoded ECDSA signature over the payload.
        requestedCertificates:
          type: object
          description: Optional certificate request set (BRC-52 format).
          additionalProperties: true

    EventEnvelope:
      type: object
      description: |
        Application-level wrapper JSON-encoded inside the BRC-103 `general`
        message payload. The `SocketServerTransport` encodes/decodes this
        transparently; application code only sees `eventName` and `data`.
      required: [eventName, data]
      properties:
        eventName:
          type: string
          description: The Socket.IO event name.
        data:
          description: The event-specific payload.

    # ----- Authentication flow -----
    AuthenticatePayload:
      type: object
      description: |
        Sent by the client on the `authenticated` event when the identity key
        was not available at connection time (fallback path). The server validates
        the key and responds with `authenticationSuccess` or `authenticationFailed`.
      properties:
        identityKey:
          $ref: "#/components/schemas/PubKeyHex"

    AuthSuccessPayload:
      type: object
      required: [status]
      properties:
        status:
          type: string
          enum: [success]

    AuthFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string
          description: Human-readable reason for failure.

    # ----- Room management -----
    JoinRoomPayload:
      type: string
      description: |
        The room ID string sent by the client on the `joinRoom` event.
        Room IDs use the convention `<recipientKey>-<messageBoxType>`.
        Example: `028d37b9...-payment_inbox`.

    JoinedRoomPayload:
      type: object
      required: [roomId]
      properties:
        roomId:
          type: string

    JoinFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    LeaveRoomPayload:
      type: string
      description: The room ID string sent by the client on the `leaveRoom` event.

    LeftRoomPayload:
      type: object
      required: [roomId]
      properties:
        roomId:
          type: string

    LeaveFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    # ----- Message sending -----
    WsSendMessagePayload:
      type: object
      description: Payload for the client-to-server `sendMessage` event.
      required: [roomId, message]
      properties:
        roomId:
          type: string
          description: |
            Target room. Format: `<recipientKey>-<messageBoxType>`.
            The server extracts the `messageBoxType` by splitting on `-` and
            taking the second part; it uses the authenticated sender key from
            `authenticatedSockets`.
        message:
          type: object
          required: [messageId, recipient, body]
          properties:
            messageId:
              type: string
              description: Unique identifier for this message (deduplication key).
            recipient:
              $ref: "#/components/schemas/PubKeyHex"
            body:
              type: string
              description: Message body string.

    WsSendMessageAckPayload:
      type: object
      description: |
        Acknowledgement emitted by the server on `sendMessageAck-{roomId}` after
        a successful `sendMessage`. Note: the event name is dynamic and includes
        the room ID used in the originating request.
      required: [status, messageId]
      properties:
        status:
          type: string
          enum: [success]
        messageId:
          type: string

    WsSendMessageBroadcastPayload:
      type: object
      description: |
        Broadcast emitted by the server on `sendMessage-{roomId}` to all
        connections in the room (including the sender). Note: the event name
        is dynamic.
      required: [sender, messageId, body]
      properties:
        sender:
          $ref: "#/components/schemas/PubKeyHex"
        messageId:
          type: string
        body:
          type: string

    MessageFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    PaymentFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    DisconnectPayload:
      type: string
      description: |
        The Socket.IO disconnect reason string (e.g. `transport close`,
        `server namespace disconnect`).

# ---------------------------------------------------------------------------
# Channels
# ---------------------------------------------------------------------------
channels:

  # -------- Low-level auth handshake (BRC-103 / Peer transport) --------
  authMessage:
    address: authMessage
    description: |
      Low-level Socket.IO event used by `SocketServerTransport` to carry
      BRC-103 `AuthMessage` frames. This event is NOT an application event;
      it is emitted and received transparently by the `Peer` class from
      `@bsv/sdk`. Application developers do not interact with this channel
      directly — they use the typed events below.

      See `packages/messaging/authsocket/src/SocketServerTransport.ts`.
    messages:
      authMessageFrame:
        name: authMessageFrame
        summary: BRC-103 auth frame (both directions — client and server).
        payload:
          $ref: "#/components/schemas/AuthMessage"

  # -------- Application authentication --------
  authenticated:
    address: authenticated
    description: |
      Fallback authentication event. The client emits this when its identity
      key was not included in the Socket.IO handshake. The server validates
      the key, updates its in-memory `authenticatedSockets` map, and emits
      `authenticationSuccess` or `authenticationFailed` in response.
    messages:
      authenticateMessage:
        name: authenticateMessage
        summary: Client sends its identity key for post-connection auth.
        payload:
          $ref: "#/components/schemas/AuthenticatePayload"

  authenticationSuccess:
    address: authenticationSuccess
    description: Emitted by the server after successful identity key validation.
    messages:
      authSuccessMessage:
        name: authSuccessMessage
        payload:
          $ref: "#/components/schemas/AuthSuccessPayload"

  authenticationFailed:
    address: authenticationFailed
    description: Emitted by the server when identity key validation fails.
    messages:
      authFailedMessage:
        name: authFailedMessage
        payload:
          $ref: "#/components/schemas/AuthFailedPayload"

  # -------- Room management --------
  joinRoom:
    address: joinRoom
    description: |
      Client requests to subscribe to a room. Only authenticated sockets may
      join rooms. The server responds with `joinedRoom` on success or
      `joinFailed` on error.

      Room ID convention: `<recipientIdentityKey>-<messageBoxType>`.
    messages:
      joinRoomMessage:
        name: joinRoomMessage
        summary: Room ID string to join.
        payload:
          $ref: "#/components/schemas/JoinRoomPayload"

  joinedRoom:
    address: joinedRoom
    description: Server confirms the client has joined the specified room.
    messages:
      joinedRoomMessage:
        name: joinedRoomMessage
        payload:
          $ref: "#/components/schemas/JoinedRoomPayload"

  joinFailed:
    address: joinFailed
    description: Emitted when `joinRoom` fails (unauthenticated or invalid roomId).
    messages:
      joinFailedMessage:
        name: joinFailedMessage
        payload:
          $ref: "#/components/schemas/JoinFailedPayload"

  leaveRoom:
    address: leaveRoom
    description: Client requests to leave a room.
    messages:
      leaveRoomMessage:
        name: leaveRoomMessage
        summary: Room ID string to leave.
        payload:
          $ref: "#/components/schemas/LeaveRoomPayload"

  leftRoom:
    address: leftRoom
    description: Server confirms the client has left the room.
    messages:
      leftRoomMessage:
        name: leftRoomMessage
        payload:
          $ref: "#/components/schemas/LeftRoomPayload"

  leaveFailed:
    address: leaveFailed
    description: Emitted when `leaveRoom` fails.
    messages:
      leaveFailedMessage:
        name: leaveFailedMessage
        payload:
          $ref: "#/components/schemas/LeaveFailedPayload"

  # -------- Message sending --------
  sendMessage:
    address: sendMessage
    description: |
      Client sends a message to a recipient via WebSocket. The server:
      1. Validates the sender is authenticated.
      2. Validates `roomId` and `message`.
      3. Creates the message box if it does not exist.
      4. Inserts the message into the database (with ON CONFLICT IGNORE dedup).
      5. Emits `sendMessageAck-{roomId}` back to the sender.
      6. Broadcasts `sendMessage-{roomId}` to all connections in the room.
    messages:
      sendMessageMessage:
        name: sendMessageMessage
        payload:
          $ref: "#/components/schemas/WsSendMessagePayload"

  sendMessageAck:
    address: "sendMessageAck-{roomId}"
    description: |
      Per-room acknowledgement emitted to the sender only after the message
      is stored. The event name is `sendMessageAck-<roomId>` where `roomId`
      matches the value in the originating `sendMessage` payload.
    parameters:
      roomId:
        description: The room ID from the originating sendMessage request.
    messages:
      sendMessageAckMessage:
        name: sendMessageAckMessage
        payload:
          $ref: "#/components/schemas/WsSendMessageAckPayload"

  sendMessageBroadcast:
    address: "sendMessage-{roomId}"
    description: |
      Broadcast emitted to all connections subscribed to `roomId` after a
      successful `sendMessage`. The event name is `sendMessage-<roomId>`.
    parameters:
      roomId:
        description: The target room ID.
    messages:
      sendMessageBroadcastMessage:
        name: sendMessageBroadcastMessage
        payload:
          $ref: "#/components/schemas/WsSendMessageBroadcastPayload"

  messageFailed:
    address: messageFailed
    description: Emitted to the sender when `sendMessage` processing fails.
    messages:
      messageFailedMessage:
        name: messageFailedMessage
        payload:
          $ref: "#/components/schemas/MessageFailedPayload"

  paymentFailed:
    address: paymentFailed
    description: |
      Emitted to unauthenticated sockets that attempt to send a message
      (same event name reused from older payment-gate logic).
    messages:
      paymentFailedMessage:
        name: paymentFailedMessage
        payload:
          $ref: "#/components/schemas/PaymentFailedPayload"

  # -------- Lifecycle --------
  disconnect:
    address: disconnect
    description: |
      Standard Socket.IO disconnect event. The server removes the socket from
      the `authenticatedSockets` map and the `peers` map in `AuthSocketServer`.
    messages:
      disconnectMessage:
        name: disconnectMessage
        payload:
          $ref: "#/components/schemas/DisconnectPayload"

# ---------------------------------------------------------------------------
# Operations
# ---------------------------------------------------------------------------
operations:

  # --- Auth handshake (transport-level, both directions) ---
  receiveAuthMessage:
    action: receive
    channel:
      $ref: "#/channels/authMessage"
    summary: Server receives an AuthMessage frame from the client during BRC-103 handshake.
    messages:
      - $ref: "#/channels/authMessage/messages/authMessageFrame"

  sendAuthMessage:
    action: send
    channel:
      $ref: "#/channels/authMessage"
    summary: Server sends an AuthMessage frame to the client during BRC-103 handshake.
    messages:
      - $ref: "#/channels/authMessage/messages/authMessageFrame"

  # --- Client authentication ---
  receiveAuthenticated:
    action: receive
    channel:
      $ref: "#/channels/authenticated"
    summary: Server receives the client's identity key on the 'authenticated' event.
    messages:
      - $ref: "#/channels/authenticated/messages/authenticateMessage"

  sendAuthenticationSuccess:
    action: send
    channel:
      $ref: "#/channels/authenticationSuccess"
    summary: Server confirms successful identity key validation.
    messages:
      - $ref: "#/channels/authenticationSuccess/messages/authSuccessMessage"

  sendAuthenticationFailed:
    action: send
    channel:
      $ref: "#/channels/authenticationFailed"
    summary: Server rejects an invalid identity key.
    messages:
      - $ref: "#/channels/authenticationFailed/messages/authFailedMessage"

  # --- Room management ---
  receiveJoinRoom:
    action: receive
    channel:
      $ref: "#/channels/joinRoom"
    summary: Server receives a room join request.
    messages:
      - $ref: "#/channels/joinRoom/messages/joinRoomMessage"

  sendJoinedRoom:
    action: send
    channel:
      $ref: "#/channels/joinedRoom"
    summary: Server confirms room join.
    messages:
      - $ref: "#/channels/joinedRoom/messages/joinedRoomMessage"

  sendJoinFailed:
    action: send
    channel:
      $ref: "#/channels/joinFailed"
    summary: Server signals room join failure.
    messages:
      - $ref: "#/channels/joinFailed/messages/joinFailedMessage"

  receiveLeaveRoom:
    action: receive
    channel:
      $ref: "#/channels/leaveRoom"
    summary: Server receives a room leave request.
    messages:
      - $ref: "#/channels/leaveRoom/messages/leaveRoomMessage"

  sendLeftRoom:
    action: send
    channel:
      $ref: "#/channels/leftRoom"
    summary: Server confirms room leave.
    messages:
      - $ref: "#/channels/leftRoom/messages/leftRoomMessage"

  sendLeaveFailed:
    action: send
    channel:
      $ref: "#/channels/leaveFailed"
    summary: Server signals room leave failure.
    messages:
      - $ref: "#/channels/leaveFailed/messages/leaveFailedMessage"

  # --- Message sending ---
  receiveSendMessage:
    action: receive
    channel:
      $ref: "#/channels/sendMessage"
    summary: Server receives a message from the client to deliver to a recipient.
    messages:
      - $ref: "#/channels/sendMessage/messages/sendMessageMessage"

  sendSendMessageAck:
    action: send
    channel:
      $ref: "#/channels/sendMessageAck"
    summary: Server acknowledges delivery of a message to the sender.
    messages:
      - $ref: "#/channels/sendMessageAck/messages/sendMessageAckMessage"

  sendSendMessageBroadcast:
    action: send
    channel:
      $ref: "#/channels/sendMessageBroadcast"
    summary: Server broadcasts a new message to all room subscribers.
    messages:
      - $ref: "#/channels/sendMessageBroadcast/messages/sendMessageBroadcastMessage"

  sendMessageFailed:
    action: send
    channel:
      $ref: "#/channels/messageFailed"
    summary: Server signals message delivery failure.
    messages:
      - $ref: "#/channels/messageFailed/messages/messageFailedMessage"

  sendPaymentFailed:
    action: send
    channel:
      $ref: "#/channels/paymentFailed"
    summary: Server signals auth/payment gate rejection.
    messages:
      - $ref: "#/channels/paymentFailed/messages/paymentFailedMessage"

  # --- Lifecycle ---
  receiveDisconnect:
    action: receive
    channel:
      $ref: "#/channels/disconnect"
    summary: Client disconnects; server cleans up in-memory state.
    messages:
      - $ref: "#/channels/disconnect/messages/disconnectMessage"
Generated deterministically from the repository source. No remote scripts, styles, fonts, or runtime dependencies.
\ No newline at end of file +AuthSocket WebSocket Protocol · AsyncAPI
AsyncAPI 3.1.0 · specs/messaging/authsocket-asyncapi.yaml

AuthSocket WebSocket Protocol

v1.0.0
AsyncAPI 3.0 specification for the `AuthSocketServer` / `AuthSocket` WebSocket channel used by the BSV MessageBox Server. ## Transport layer `AuthSocketServer` wraps Socket.IO and sits on top of an HTTP server. All Socket.IO events are standard Socket.IO framing; this spec describes the *application-level* event names and payload shapes that flow over the Socket.IO connection. ## BRC-103 mutual authentication Every Socket.IO connection undergoes BRC-103 (`Peer`) handshake before application events are exchanged. The handshake is carried on the **`authMessage`** event using the `AuthMessage` envelope defined in the `@bsv/sdk` `Transport` interface. Once the handshake succeeds the peer's `identityKey` (compressed secp256k1 public key, 66-char hex) is known server-side and stored in memory for the lifetime of the connection. ## Application events After authentication the server emits and listens for the events described below. All application payloads are serialized as JSON inside the BRC-103 `general` message (`Peer.toPeer`). The transport layer (`SocketServerTransport`) wraps them in an `{ eventName, data }` envelope before signing. Source of truth: - `packages/messaging/authsocket/src/AuthSocketServer.ts` - `packages/messaging/authsocket/src/SocketServerTransport.ts` - `packages/messaging/message-box-server/src/index.ts`
2Servers16Channels17Operations0Messages18Schemas

Servers

2

production

wss://messagebox.babbage.systems/

Production MessageBox WebSocket endpoint (Socket.IO over WSS).
host
messagebox.babbage.systems
pathname
/
protocol
wss
description
Production MessageBox WebSocket endpoint (Socket.IO over WSS).

local

ws://localhost:{port}/

Local development Socket.IO server.
host
localhost:{port}
pathname
/
protocol
ws
description
Local development Socket.IO server.
variables
1 field
port
2 fields
default
5001
description
HTTP port the MessageBox Server listens on.

Channels

16

authMessage

authMessage

Low-level Socket.IO event used by `SocketServerTransport` to carry BRC-103 `AuthMessage` frames. This event is NOT an application event; it is emitted and received transparently by the `Peer` class from `@bsv/sdk`. Application developers do not interact with this channel directly — they use the typed events below. See `packages/messaging/authsocket/src/SocketServerTransport.ts`.
address
authMessage
description
Low-level Socket.IO event used by `SocketServerTransport` to carry BRC-103 `AuthMessage` frames. This event is NOT an application event; it is emitted and received transparently by the `Peer` class from `@bsv/sdk`. Application developers do not interact with this channel directly — they use the typed events below. See `packages/messaging/authsocket/src/SocketServerTransport.ts`.
messages
1 field
authMessageFrame
3 fields
name
authMessageFrame
summary
BRC-103 auth frame (both directions — client and server).
payload
1 field
$ref
#/components/schemas/AuthMessage

authenticated

authenticated

Fallback authentication event. The client emits this when its identity key was not included in the Socket.IO handshake. The server validates the key, updates its in-memory `authenticatedSockets` map, and emits `authenticationSuccess` or `authenticationFailed` in response.
address
authenticated
description
Fallback authentication event. The client emits this when its identity key was not included in the Socket.IO handshake. The server validates the key, updates its in-memory `authenticatedSockets` map, and emits `authenticationSuccess` or `authenticationFailed` in response.
messages
1 field
authenticateMessage
3 fields
name
authenticateMessage
summary
Client sends its identity key for post-connection auth.
payload
1 field
$ref
#/components/schemas/AuthenticatePayload

authenticationSuccess

authenticationSuccess

Emitted by the server after successful identity key validation.
address
authenticationSuccess
description
Emitted by the server after successful identity key validation.
messages
1 field
authSuccessMessage
2 fields
name
authSuccessMessage
payload
1 field
$ref
#/components/schemas/AuthSuccessPayload

authenticationFailed

authenticationFailed

Emitted by the server when identity key validation fails.
address
authenticationFailed
description
Emitted by the server when identity key validation fails.
messages
1 field
authFailedMessage
2 fields
name
authFailedMessage
payload
1 field
$ref
#/components/schemas/AuthFailedPayload

joinRoom

joinRoom

Client requests to subscribe to a room. Only authenticated sockets may join rooms. The server responds with `joinedRoom` on success or `joinFailed` on error. Room ID convention: `<recipientIdentityKey>-<messageBoxType>`.
address
joinRoom
description
Client requests to subscribe to a room. Only authenticated sockets may join rooms. The server responds with `joinedRoom` on success or `joinFailed` on error. Room ID convention: `<recipientIdentityKey>-<messageBoxType>`.
messages
1 field
joinRoomMessage
3 fields
name
joinRoomMessage
summary
Room ID string to join.
payload
1 field
$ref
#/components/schemas/JoinRoomPayload

joinedRoom

joinedRoom

Server confirms the client has joined the specified room.
address
joinedRoom
description
Server confirms the client has joined the specified room.
messages
1 field
joinedRoomMessage
2 fields
name
joinedRoomMessage
payload
1 field
$ref
#/components/schemas/JoinedRoomPayload

joinFailed

joinFailed

Emitted when `joinRoom` fails (unauthenticated or invalid roomId).
address
joinFailed
description
Emitted when `joinRoom` fails (unauthenticated or invalid roomId).
messages
1 field
joinFailedMessage
2 fields
name
joinFailedMessage
payload
1 field
$ref
#/components/schemas/JoinFailedPayload

leaveRoom

leaveRoom

Client requests to leave a room.
address
leaveRoom
description
Client requests to leave a room.
messages
1 field
leaveRoomMessage
3 fields
name
leaveRoomMessage
summary
Room ID string to leave.
payload
1 field
$ref
#/components/schemas/LeaveRoomPayload

leftRoom

leftRoom

Server confirms the client has left the room.
address
leftRoom
description
Server confirms the client has left the room.
messages
1 field
leftRoomMessage
2 fields
name
leftRoomMessage
payload
1 field
$ref
#/components/schemas/LeftRoomPayload

leaveFailed

leaveFailed

Emitted when `leaveRoom` fails.
address
leaveFailed
description
Emitted when `leaveRoom` fails.
messages
1 field
leaveFailedMessage
2 fields
name
leaveFailedMessage
payload
1 field
$ref
#/components/schemas/LeaveFailedPayload

sendMessage

sendMessage

Client sends a message to a recipient via WebSocket when operator monetization is disabled. Paid servers return an error acknowledgement that instructs current clients to use their AuthFetch HTTP fallback. The unpriced WebSocket path: 1. Validates the sender is authenticated. 2. Validates `roomId` and `message`. 3. Creates the message box if it does not exist. 4. Inserts the message into the database (with ON CONFLICT IGNORE dedup). 5. Emits `sendMessageAck-{roomId}` back to the sender. 6. Broadcasts `sendMessage-{roomId}` to all connections in the room.
address
sendMessage
description
Client sends a message to a recipient via WebSocket when operator monetization is disabled. Paid servers return an error acknowledgement that instructs current clients to use their AuthFetch HTTP fallback. The unpriced WebSocket path: 1. Validates the sender is authenticated. 2. Validates `roomId` and `message`. 3. Creates the message box if it does not exist. 4. Inserts the message into the database (with ON CONFLICT IGNORE dedup). 5. Emits `sendMessageAck-{roomId}` back to the sender. 6. Broadcasts `sendMessage-{roomId}` to all connections in the room.
messages
1 field
sendMessageMessage
2 fields
name
sendMessageMessage
payload
1 field
$ref
#/components/schemas/WsSendMessagePayload

sendMessageAck

sendMessageAck-{roomId}

Per-room acknowledgement emitted to the sender after the message is stored or when the request must fall back to AuthFetch. The event name is `sendMessageAck-<roomId>` where `roomId` matches the value in the originating `sendMessage` payload.
address
sendMessageAck-{roomId}
description
Per-room acknowledgement emitted to the sender after the message is stored or when the request must fall back to AuthFetch. The event name is `sendMessageAck-<roomId>` where `roomId` matches the value in the originating `sendMessage` payload.
parameters
1 field
roomId
1 field
description
The room ID from the originating sendMessage request.
messages
1 field
sendMessageAckMessage
2 fields
name
sendMessageAckMessage
payload
1 field
$ref
#/components/schemas/WsSendMessageAckPayload

sendMessageBroadcast

sendMessage-{roomId}

Broadcast emitted to all connections subscribed to `roomId` after a successful `sendMessage`. The event name is `sendMessage-<roomId>`.
address
sendMessage-{roomId}
description
Broadcast emitted to all connections subscribed to `roomId` after a successful `sendMessage`. The event name is `sendMessage-<roomId>`.
parameters
1 field
roomId
1 field
description
The target room ID.
messages
1 field
sendMessageBroadcastMessage
2 fields
name
sendMessageBroadcastMessage
payload
1 field
$ref
#/components/schemas/WsSendMessageBroadcastPayload

messageFailed

messageFailed

Emitted to the sender when `sendMessage` processing fails.
address
messageFailed
description
Emitted to the sender when `sendMessage` processing fails.
messages
1 field
messageFailedMessage
2 fields
name
messageFailedMessage
payload
1 field
$ref
#/components/schemas/MessageFailedPayload

paymentFailed

paymentFailed

Emitted to unauthenticated sockets that attempt to send a message (same event name reused from older payment-gate logic).
address
paymentFailed
description
Emitted to unauthenticated sockets that attempt to send a message (same event name reused from older payment-gate logic).
messages
1 field
paymentFailedMessage
2 fields
name
paymentFailedMessage
payload
1 field
$ref
#/components/schemas/PaymentFailedPayload

disconnect

disconnect

Standard Socket.IO disconnect event. The server removes the socket from the `authenticatedSockets` map and the `peers` map in `AuthSocketServer`.
address
disconnect
description
Standard Socket.IO disconnect event. The server removes the socket from the `authenticatedSockets` map and the `peers` map in `AuthSocketServer`.
messages
1 field
disconnectMessage
2 fields
name
disconnectMessage
payload
1 field
$ref
#/components/schemas/DisconnectPayload

Operations

17

receiveAuthMessage

receive
Server receives an AuthMessage frame from the client during BRC-103 handshake.
action
receive
channel
1 field
$ref
#/channels/authMessage
summary
Server receives an AuthMessage frame from the client during BRC-103 handshake.
messages
1 item
  1. $ref
    #/channels/authMessage/messages/authMessageFrame

sendAuthMessage

send
Server sends an AuthMessage frame to the client during BRC-103 handshake.
action
send
channel
1 field
$ref
#/channels/authMessage
summary
Server sends an AuthMessage frame to the client during BRC-103 handshake.
messages
1 item
  1. $ref
    #/channels/authMessage/messages/authMessageFrame

receiveAuthenticated

receive
Server receives the client's identity key on the 'authenticated' event.
action
receive
channel
1 field
$ref
#/channels/authenticated
summary
Server receives the client's identity key on the 'authenticated' event.
messages
1 item
  1. $ref
    #/channels/authenticated/messages/authenticateMessage

sendAuthenticationSuccess

send
Server confirms successful identity key validation.
action
send
channel
1 field
$ref
#/channels/authenticationSuccess
summary
Server confirms successful identity key validation.
messages
1 item
  1. $ref
    #/channels/authenticationSuccess/messages/authSuccessMessage

sendAuthenticationFailed

send
Server rejects an invalid identity key.
action
send
channel
1 field
$ref
#/channels/authenticationFailed
summary
Server rejects an invalid identity key.
messages
1 item
  1. $ref
    #/channels/authenticationFailed/messages/authFailedMessage

receiveJoinRoom

receive
Server receives a room join request.
action
receive
channel
1 field
$ref
#/channels/joinRoom
summary
Server receives a room join request.
messages
1 item
  1. $ref
    #/channels/joinRoom/messages/joinRoomMessage

sendJoinedRoom

send
Server confirms room join.
action
send
channel
1 field
$ref
#/channels/joinedRoom
summary
Server confirms room join.
messages
1 item
  1. $ref
    #/channels/joinedRoom/messages/joinedRoomMessage

sendJoinFailed

send
Server signals room join failure.
action
send
channel
1 field
$ref
#/channels/joinFailed
summary
Server signals room join failure.
messages
1 item
  1. $ref
    #/channels/joinFailed/messages/joinFailedMessage

receiveLeaveRoom

receive
Server receives a room leave request.
action
receive
channel
1 field
$ref
#/channels/leaveRoom
summary
Server receives a room leave request.
messages
1 item
  1. $ref
    #/channels/leaveRoom/messages/leaveRoomMessage

sendLeftRoom

send
Server confirms room leave.
action
send
channel
1 field
$ref
#/channels/leftRoom
summary
Server confirms room leave.
messages
1 item
  1. $ref
    #/channels/leftRoom/messages/leftRoomMessage

sendLeaveFailed

send
Server signals room leave failure.
action
send
channel
1 field
$ref
#/channels/leaveFailed
summary
Server signals room leave failure.
messages
1 item
  1. $ref
    #/channels/leaveFailed/messages/leaveFailedMessage

receiveSendMessage

receive
Server receives a message from the client to deliver to a recipient.
action
receive
channel
1 field
$ref
#/channels/sendMessage
summary
Server receives a message from the client to deliver to a recipient.
messages
1 item
  1. $ref
    #/channels/sendMessage/messages/sendMessageMessage

sendSendMessageAck

send
Server acknowledges delivery of a message to the sender.
action
send
channel
1 field
$ref
#/channels/sendMessageAck
summary
Server acknowledges delivery of a message to the sender.
messages
1 item
  1. $ref
    #/channels/sendMessageAck/messages/sendMessageAckMessage

sendSendMessageBroadcast

send
Server broadcasts a new message to all room subscribers.
action
send
channel
1 field
$ref
#/channels/sendMessageBroadcast
summary
Server broadcasts a new message to all room subscribers.
messages
1 item
  1. $ref
    #/channels/sendMessageBroadcast/messages/sendMessageBroadcastMessage

sendMessageFailed

send
Server signals message delivery failure.
action
send
channel
1 field
$ref
#/channels/messageFailed
summary
Server signals message delivery failure.
messages
1 item
  1. $ref
    #/channels/messageFailed/messages/messageFailedMessage

sendPaymentFailed

send
Server signals auth/payment gate rejection.
action
send
channel
1 field
$ref
#/channels/paymentFailed
summary
Server signals auth/payment gate rejection.
messages
1 item
  1. $ref
    #/channels/paymentFailed/messages/paymentFailedMessage

receiveDisconnect

receive
Client disconnects; server cleans up in-memory state.
action
receive
channel
1 field
$ref
#/channels/disconnect
summary
Client disconnects; server cleans up in-memory state.
messages
1 item
  1. $ref
    #/channels/disconnect/messages/disconnectMessage

Schemas

18

PubKeyHex

Compressed secp256k1 public key, 66 hex characters.
type
string
pattern
^0[23][0-9a-fA-F]{64}$
description
Compressed secp256k1 public key, 66 hex characters.

AuthMessage

BRC-103 auth envelope as defined in `@bsv/sdk`. Carried on the low-level `authMessage` Socket.IO event. Not an application-level event.
type
object
description
BRC-103 auth envelope as defined in `@bsv/sdk`. Carried on the low-level `authMessage` Socket.IO event. Not an application-level event.
required
3 items
  1. messageType
  2. version
  3. identityKey
properties
9 fields
messageType
3 fields
type
string
enum
3 items
  1. initialRequest
  2. initialResponse
  3. general
description
- `initialRequest` — first handshake message from initiating peer - `initialResponse` — server's challenge response (includes nonce, signature) - `general` — signed application payload after handshake
version
2 fields
type
string
description
Auth protocol version string.
identityKey
1 field
$ref
#/components/schemas/PubKeyHex
nonce
2 fields
type
string
description
Fresh random nonce (base64) generated by the sender.
yourNonce
2 fields
type
string
description
Echo of the peer's nonce from the previous message.
initialNonce
2 fields
type
string
description
Present in `initialRequest`; absent in subsequent messages.
payload
3 fields
type
array
items
1 field
type
integer
description
Signed application payload (byte array). Empty for handshake messages.
signature
3 fields
type
array
items
1 field
type
integer
description
DER-encoded ECDSA signature over the payload.
requestedCertificates
3 fields
type
object
description
Optional certificate request set (BRC-52 format).
additionalProperties
true

EventEnvelope

Application-level wrapper JSON-encoded inside the BRC-103 `general` message payload. The `SocketServerTransport` encodes/decodes this transparently; application code only sees `eventName` and `data`.
type
object
description
Application-level wrapper JSON-encoded inside the BRC-103 `general` message payload. The `SocketServerTransport` encodes/decodes this transparently; application code only sees `eventName` and `data`.
required
2 items
  1. eventName
  2. data
properties
2 fields
eventName
2 fields
type
string
description
The Socket.IO event name.
data
1 field
description
The event-specific payload.

AuthenticatePayload

Sent by the client on the `authenticated` event when the identity key was not available at connection time (fallback path). The server validates the key and responds with `authenticationSuccess` or `authenticationFailed`.
type
object
description
Sent by the client on the `authenticated` event when the identity key was not available at connection time (fallback path). The server validates the key and responds with `authenticationSuccess` or `authenticationFailed`.
properties
1 field
identityKey
1 field
$ref
#/components/schemas/PubKeyHex

AuthSuccessPayload

type
object
required
1 item
  1. status
properties
1 field
status
2 fields
type
string
enum
1 item
  1. success

AuthFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
2 fields
type
string
description
Human-readable reason for failure.

JoinRoomPayload

The room ID string sent by the client on the `joinRoom` event. Room IDs use the convention `<recipientKey>-<messageBoxType>`. Example: `028d37b9...-payment_inbox`.
type
string
description
The room ID string sent by the client on the `joinRoom` event. Room IDs use the convention `<recipientKey>-<messageBoxType>`. Example: `028d37b9...-payment_inbox`.

JoinedRoomPayload

type
object
required
1 item
  1. roomId
properties
1 field
roomId
1 field
type
string

JoinFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

LeaveRoomPayload

The room ID string sent by the client on the `leaveRoom` event.
type
string
description
The room ID string sent by the client on the `leaveRoom` event.

LeftRoomPayload

type
object
required
1 item
  1. roomId
properties
1 field
roomId
1 field
type
string

LeaveFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

WsSendMessagePayload

Payload for the client-to-server `sendMessage` event.
type
object
description
Payload for the client-to-server `sendMessage` event.
required
2 items
  1. roomId
  2. message
properties
2 fields
roomId
2 fields
type
string
description
Target room. Format: `<recipientKey>-<messageBoxType>`. The server removes the exact recipient-key prefix and uses the authenticated sender key from `authenticatedSockets`.
message
3 fields
type
object
required
3 items
  1. messageId
  2. recipient
  3. body
properties
3 fields
messageId
2 fields
type
string
description
Unique identifier for this message (deduplication key).
recipient
1 field
$ref
#/components/schemas/PubKeyHex
body
2 fields
type
string
description
Message body string.

WsSendMessageAckPayload

Acknowledgement emitted by the server on `sendMessageAck-{roomId}`. A successful write includes `messageId`. An error includes `code`; paid servers use `ERR_PAYMENT_REQUIRES_AUTHFETCH` so compatible clients retry the send through the BRC-105 AuthFetch HTTP path.
type
object
description
Acknowledgement emitted by the server on `sendMessageAck-{roomId}`. A successful write includes `messageId`. An error includes `code`; paid servers use `ERR_PAYMENT_REQUIRES_AUTHFETCH` so compatible clients retry the send through the BRC-105 AuthFetch HTTP path.
required
1 item
  1. status
properties
3 fields
status
2 fields
type
string
enum
2 items
  1. success
  2. error
messageId
1 field
type
string
code
1 field
type
string

WsSendMessageBroadcastPayload

Broadcast emitted by the server on `sendMessage-{roomId}` to all connections in the room (including the sender). Note: the event name is dynamic.
type
object
description
Broadcast emitted by the server on `sendMessage-{roomId}` to all connections in the room (including the sender). Note: the event name is dynamic.
required
3 items
  1. sender
  2. messageId
  3. body
properties
3 fields
sender
1 field
$ref
#/components/schemas/PubKeyHex
messageId
1 field
type
string
body
1 field
type
string

MessageFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

PaymentFailedPayload

type
object
required
1 item
  1. reason
properties
1 field
reason
1 field
type
string

DisconnectPayload

The Socket.IO disconnect reason string (e.g. `transport close`, `server namespace disconnect`).
type
string
description
The Socket.IO disconnect reason string (e.g. `transport close`, `server namespace disconnect`).
Raw YAML source
asyncapi: "3.1.0"

info:
  title: AuthSocket WebSocket Protocol
  version: "1.0.0"
  description: |
    AsyncAPI 3.0 specification for the `AuthSocketServer` / `AuthSocket`
    WebSocket channel used by the BSV MessageBox Server.

    ## Transport layer

    `AuthSocketServer` wraps Socket.IO and sits on top of an HTTP server.
    All Socket.IO events are standard Socket.IO framing; this spec describes
    the *application-level* event names and payload shapes that flow over the
    Socket.IO connection.

    ## BRC-103 mutual authentication

    Every Socket.IO connection undergoes BRC-103 (`Peer`) handshake before
    application events are exchanged. The handshake is carried on the
    **`authMessage`** event using the `AuthMessage` envelope defined in the
    `@bsv/sdk` `Transport` interface.

    Once the handshake succeeds the peer's `identityKey` (compressed secp256k1
    public key, 66-char hex) is known server-side and stored in memory for the
    lifetime of the connection.

    ## Application events

    After authentication the server emits and listens for the events described
    below. All application payloads are serialized as JSON inside the
    BRC-103 `general` message (`Peer.toPeer`). The transport layer
    (`SocketServerTransport`) wraps them in an `{ eventName, data }` envelope
    before signing.

    Source of truth:
    - `packages/messaging/authsocket/src/AuthSocketServer.ts`
    - `packages/messaging/authsocket/src/SocketServerTransport.ts`
    - `packages/messaging/message-box-server/src/index.ts`

servers:
  production:
    host: "messagebox.babbage.systems"
    pathname: "/"
    protocol: wss
    description: Production MessageBox WebSocket endpoint (Socket.IO over WSS).
  local:
    host: "localhost:{port}"
    pathname: "/"
    protocol: ws
    description: Local development Socket.IO server.
    variables:
      port:
        default: "5001"
        description: HTTP port the MessageBox Server listens on.

# ---------------------------------------------------------------------------
# Components
# ---------------------------------------------------------------------------
components:
  schemas:
    PubKeyHex:
      type: string
      pattern: "^0[23][0-9a-fA-F]{64}$"
      description: Compressed secp256k1 public key, 66 hex characters.

    AuthMessage:
      type: object
      description: |
        BRC-103 auth envelope as defined in `@bsv/sdk`. Carried on the
        low-level `authMessage` Socket.IO event. Not an application-level event.
      required: [messageType, version, identityKey]
      properties:
        messageType:
          type: string
          enum: [initialRequest, initialResponse, general]
          description: |
            - `initialRequest`  — first handshake message from initiating peer
            - `initialResponse` — server's challenge response (includes nonce, signature)
            - `general`         — signed application payload after handshake
        version:
          type: string
          description: Auth protocol version string.
        identityKey:
          $ref: "#/components/schemas/PubKeyHex"
        nonce:
          type: string
          description: Fresh random nonce (base64) generated by the sender.
        yourNonce:
          type: string
          description: Echo of the peer's nonce from the previous message.
        initialNonce:
          type: string
          description: Present in `initialRequest`; absent in subsequent messages.
        payload:
          type: array
          items:
            type: integer
          description: Signed application payload (byte array). Empty for handshake messages.
        signature:
          type: array
          items:
            type: integer
          description: DER-encoded ECDSA signature over the payload.
        requestedCertificates:
          type: object
          description: Optional certificate request set (BRC-52 format).
          additionalProperties: true

    EventEnvelope:
      type: object
      description: |
        Application-level wrapper JSON-encoded inside the BRC-103 `general`
        message payload. The `SocketServerTransport` encodes/decodes this
        transparently; application code only sees `eventName` and `data`.
      required: [eventName, data]
      properties:
        eventName:
          type: string
          description: The Socket.IO event name.
        data:
          description: The event-specific payload.

    # ----- Authentication flow -----
    AuthenticatePayload:
      type: object
      description: |
        Sent by the client on the `authenticated` event when the identity key
        was not available at connection time (fallback path). The server validates
        the key and responds with `authenticationSuccess` or `authenticationFailed`.
      properties:
        identityKey:
          $ref: "#/components/schemas/PubKeyHex"

    AuthSuccessPayload:
      type: object
      required: [status]
      properties:
        status:
          type: string
          enum: [success]

    AuthFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string
          description: Human-readable reason for failure.

    # ----- Room management -----
    JoinRoomPayload:
      type: string
      description: |
        The room ID string sent by the client on the `joinRoom` event.
        Room IDs use the convention `<recipientKey>-<messageBoxType>`.
        Example: `028d37b9...-payment_inbox`.

    JoinedRoomPayload:
      type: object
      required: [roomId]
      properties:
        roomId:
          type: string

    JoinFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    LeaveRoomPayload:
      type: string
      description: The room ID string sent by the client on the `leaveRoom` event.

    LeftRoomPayload:
      type: object
      required: [roomId]
      properties:
        roomId:
          type: string

    LeaveFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    # ----- Message sending -----
    WsSendMessagePayload:
      type: object
      description: Payload for the client-to-server `sendMessage` event.
      required: [roomId, message]
      properties:
        roomId:
          type: string
          description: |
            Target room. Format: `<recipientKey>-<messageBoxType>`.
            The server removes the exact recipient-key prefix and uses the
            authenticated sender key from `authenticatedSockets`.
        message:
          type: object
          required: [messageId, recipient, body]
          properties:
            messageId:
              type: string
              description: Unique identifier for this message (deduplication key).
            recipient:
              $ref: "#/components/schemas/PubKeyHex"
            body:
              type: string
              description: Message body string.

    WsSendMessageAckPayload:
      type: object
      description: |
        Acknowledgement emitted by the server on `sendMessageAck-{roomId}`. A
        successful write includes `messageId`. An error includes `code`; paid
        servers use `ERR_PAYMENT_REQUIRES_AUTHFETCH` so compatible clients retry
        the send through the BRC-105 AuthFetch HTTP path.
      required: [status]
      properties:
        status:
          type: string
          enum: [success, error]
        messageId:
          type: string
        code:
          type: string

    WsSendMessageBroadcastPayload:
      type: object
      description: |
        Broadcast emitted by the server on `sendMessage-{roomId}` to all
        connections in the room (including the sender). Note: the event name
        is dynamic.
      required: [sender, messageId, body]
      properties:
        sender:
          $ref: "#/components/schemas/PubKeyHex"
        messageId:
          type: string
        body:
          type: string

    MessageFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    PaymentFailedPayload:
      type: object
      required: [reason]
      properties:
        reason:
          type: string

    DisconnectPayload:
      type: string
      description: |
        The Socket.IO disconnect reason string (e.g. `transport close`,
        `server namespace disconnect`).

# ---------------------------------------------------------------------------
# Channels
# ---------------------------------------------------------------------------
channels:

  # -------- Low-level auth handshake (BRC-103 / Peer transport) --------
  authMessage:
    address: authMessage
    description: |
      Low-level Socket.IO event used by `SocketServerTransport` to carry
      BRC-103 `AuthMessage` frames. This event is NOT an application event;
      it is emitted and received transparently by the `Peer` class from
      `@bsv/sdk`. Application developers do not interact with this channel
      directly — they use the typed events below.

      See `packages/messaging/authsocket/src/SocketServerTransport.ts`.
    messages:
      authMessageFrame:
        name: authMessageFrame
        summary: BRC-103 auth frame (both directions — client and server).
        payload:
          $ref: "#/components/schemas/AuthMessage"

  # -------- Application authentication --------
  authenticated:
    address: authenticated
    description: |
      Fallback authentication event. The client emits this when its identity
      key was not included in the Socket.IO handshake. The server validates
      the key, updates its in-memory `authenticatedSockets` map, and emits
      `authenticationSuccess` or `authenticationFailed` in response.
    messages:
      authenticateMessage:
        name: authenticateMessage
        summary: Client sends its identity key for post-connection auth.
        payload:
          $ref: "#/components/schemas/AuthenticatePayload"

  authenticationSuccess:
    address: authenticationSuccess
    description: Emitted by the server after successful identity key validation.
    messages:
      authSuccessMessage:
        name: authSuccessMessage
        payload:
          $ref: "#/components/schemas/AuthSuccessPayload"

  authenticationFailed:
    address: authenticationFailed
    description: Emitted by the server when identity key validation fails.
    messages:
      authFailedMessage:
        name: authFailedMessage
        payload:
          $ref: "#/components/schemas/AuthFailedPayload"

  # -------- Room management --------
  joinRoom:
    address: joinRoom
    description: |
      Client requests to subscribe to a room. Only authenticated sockets may
      join rooms. The server responds with `joinedRoom` on success or
      `joinFailed` on error.

      Room ID convention: `<recipientIdentityKey>-<messageBoxType>`.
    messages:
      joinRoomMessage:
        name: joinRoomMessage
        summary: Room ID string to join.
        payload:
          $ref: "#/components/schemas/JoinRoomPayload"

  joinedRoom:
    address: joinedRoom
    description: Server confirms the client has joined the specified room.
    messages:
      joinedRoomMessage:
        name: joinedRoomMessage
        payload:
          $ref: "#/components/schemas/JoinedRoomPayload"

  joinFailed:
    address: joinFailed
    description: Emitted when `joinRoom` fails (unauthenticated or invalid roomId).
    messages:
      joinFailedMessage:
        name: joinFailedMessage
        payload:
          $ref: "#/components/schemas/JoinFailedPayload"

  leaveRoom:
    address: leaveRoom
    description: Client requests to leave a room.
    messages:
      leaveRoomMessage:
        name: leaveRoomMessage
        summary: Room ID string to leave.
        payload:
          $ref: "#/components/schemas/LeaveRoomPayload"

  leftRoom:
    address: leftRoom
    description: Server confirms the client has left the room.
    messages:
      leftRoomMessage:
        name: leftRoomMessage
        payload:
          $ref: "#/components/schemas/LeftRoomPayload"

  leaveFailed:
    address: leaveFailed
    description: Emitted when `leaveRoom` fails.
    messages:
      leaveFailedMessage:
        name: leaveFailedMessage
        payload:
          $ref: "#/components/schemas/LeaveFailedPayload"

  # -------- Message sending --------
  sendMessage:
    address: sendMessage
    description: |
      Client sends a message to a recipient via WebSocket when operator
      monetization is disabled. Paid servers return an error acknowledgement
      that instructs current clients to use their AuthFetch HTTP fallback. The
      unpriced WebSocket path:
      1. Validates the sender is authenticated.
      2. Validates `roomId` and `message`.
      3. Creates the message box if it does not exist.
      4. Inserts the message into the database (with ON CONFLICT IGNORE dedup).
      5. Emits `sendMessageAck-{roomId}` back to the sender.
      6. Broadcasts `sendMessage-{roomId}` to all connections in the room.
    messages:
      sendMessageMessage:
        name: sendMessageMessage
        payload:
          $ref: "#/components/schemas/WsSendMessagePayload"

  sendMessageAck:
    address: "sendMessageAck-{roomId}"
    description: |
      Per-room acknowledgement emitted to the sender after the message is
      stored or when the request must fall back to AuthFetch. The event name is
      `sendMessageAck-<roomId>` where `roomId` matches the value in the
      originating `sendMessage` payload.
    parameters:
      roomId:
        description: The room ID from the originating sendMessage request.
    messages:
      sendMessageAckMessage:
        name: sendMessageAckMessage
        payload:
          $ref: "#/components/schemas/WsSendMessageAckPayload"

  sendMessageBroadcast:
    address: "sendMessage-{roomId}"
    description: |
      Broadcast emitted to all connections subscribed to `roomId` after a
      successful `sendMessage`. The event name is `sendMessage-<roomId>`.
    parameters:
      roomId:
        description: The target room ID.
    messages:
      sendMessageBroadcastMessage:
        name: sendMessageBroadcastMessage
        payload:
          $ref: "#/components/schemas/WsSendMessageBroadcastPayload"

  messageFailed:
    address: messageFailed
    description: Emitted to the sender when `sendMessage` processing fails.
    messages:
      messageFailedMessage:
        name: messageFailedMessage
        payload:
          $ref: "#/components/schemas/MessageFailedPayload"

  paymentFailed:
    address: paymentFailed
    description: |
      Emitted to unauthenticated sockets that attempt to send a message
      (same event name reused from older payment-gate logic).
    messages:
      paymentFailedMessage:
        name: paymentFailedMessage
        payload:
          $ref: "#/components/schemas/PaymentFailedPayload"

  # -------- Lifecycle --------
  disconnect:
    address: disconnect
    description: |
      Standard Socket.IO disconnect event. The server removes the socket from
      the `authenticatedSockets` map and the `peers` map in `AuthSocketServer`.
    messages:
      disconnectMessage:
        name: disconnectMessage
        payload:
          $ref: "#/components/schemas/DisconnectPayload"

# ---------------------------------------------------------------------------
# Operations
# ---------------------------------------------------------------------------
operations:

  # --- Auth handshake (transport-level, both directions) ---
  receiveAuthMessage:
    action: receive
    channel:
      $ref: "#/channels/authMessage"
    summary: Server receives an AuthMessage frame from the client during BRC-103 handshake.
    messages:
      - $ref: "#/channels/authMessage/messages/authMessageFrame"

  sendAuthMessage:
    action: send
    channel:
      $ref: "#/channels/authMessage"
    summary: Server sends an AuthMessage frame to the client during BRC-103 handshake.
    messages:
      - $ref: "#/channels/authMessage/messages/authMessageFrame"

  # --- Client authentication ---
  receiveAuthenticated:
    action: receive
    channel:
      $ref: "#/channels/authenticated"
    summary: Server receives the client's identity key on the 'authenticated' event.
    messages:
      - $ref: "#/channels/authenticated/messages/authenticateMessage"

  sendAuthenticationSuccess:
    action: send
    channel:
      $ref: "#/channels/authenticationSuccess"
    summary: Server confirms successful identity key validation.
    messages:
      - $ref: "#/channels/authenticationSuccess/messages/authSuccessMessage"

  sendAuthenticationFailed:
    action: send
    channel:
      $ref: "#/channels/authenticationFailed"
    summary: Server rejects an invalid identity key.
    messages:
      - $ref: "#/channels/authenticationFailed/messages/authFailedMessage"

  # --- Room management ---
  receiveJoinRoom:
    action: receive
    channel:
      $ref: "#/channels/joinRoom"
    summary: Server receives a room join request.
    messages:
      - $ref: "#/channels/joinRoom/messages/joinRoomMessage"

  sendJoinedRoom:
    action: send
    channel:
      $ref: "#/channels/joinedRoom"
    summary: Server confirms room join.
    messages:
      - $ref: "#/channels/joinedRoom/messages/joinedRoomMessage"

  sendJoinFailed:
    action: send
    channel:
      $ref: "#/channels/joinFailed"
    summary: Server signals room join failure.
    messages:
      - $ref: "#/channels/joinFailed/messages/joinFailedMessage"

  receiveLeaveRoom:
    action: receive
    channel:
      $ref: "#/channels/leaveRoom"
    summary: Server receives a room leave request.
    messages:
      - $ref: "#/channels/leaveRoom/messages/leaveRoomMessage"

  sendLeftRoom:
    action: send
    channel:
      $ref: "#/channels/leftRoom"
    summary: Server confirms room leave.
    messages:
      - $ref: "#/channels/leftRoom/messages/leftRoomMessage"

  sendLeaveFailed:
    action: send
    channel:
      $ref: "#/channels/leaveFailed"
    summary: Server signals room leave failure.
    messages:
      - $ref: "#/channels/leaveFailed/messages/leaveFailedMessage"

  # --- Message sending ---
  receiveSendMessage:
    action: receive
    channel:
      $ref: "#/channels/sendMessage"
    summary: Server receives a message from the client to deliver to a recipient.
    messages:
      - $ref: "#/channels/sendMessage/messages/sendMessageMessage"

  sendSendMessageAck:
    action: send
    channel:
      $ref: "#/channels/sendMessageAck"
    summary: Server acknowledges delivery of a message to the sender.
    messages:
      - $ref: "#/channels/sendMessageAck/messages/sendMessageAckMessage"

  sendSendMessageBroadcast:
    action: send
    channel:
      $ref: "#/channels/sendMessageBroadcast"
    summary: Server broadcasts a new message to all room subscribers.
    messages:
      - $ref: "#/channels/sendMessageBroadcast/messages/sendMessageBroadcastMessage"

  sendMessageFailed:
    action: send
    channel:
      $ref: "#/channels/messageFailed"
    summary: Server signals message delivery failure.
    messages:
      - $ref: "#/channels/messageFailed/messages/messageFailedMessage"

  sendPaymentFailed:
    action: send
    channel:
      $ref: "#/channels/paymentFailed"
    summary: Server signals auth/payment gate rejection.
    messages:
      - $ref: "#/channels/paymentFailed/messages/paymentFailedMessage"

  # --- Lifecycle ---
  receiveDisconnect:
    action: receive
    channel:
      $ref: "#/channels/disconnect"
    summary: Client disconnects; server cleans up in-memory state.
    messages:
      - $ref: "#/channels/disconnect/messages/disconnectMessage"
Generated deterministically from the repository source. No remote scripts, styles, fonts, or runtime dependencies.
\ No newline at end of file diff --git a/docs/examples/kubernetes/resource-profiles/message-box-hpa.yaml b/docs/examples/kubernetes/resource-profiles/message-box-hpa.yaml new file mode 100644 index 000000000..0eb3b33fb --- /dev/null +++ b/docs/examples/kubernetes/resource-profiles/message-box-hpa.yaml @@ -0,0 +1,40 @@ +# Prerequisites: shared MySQL sessions/replay/quota state and a shared gateway +# rate limit. Disable WebSockets or provide sticky routing / shared Socket.IO +# pub-sub before increasing replicas. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: message-box-server +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: message-box-server + minReplicas: 2 + maxReplicas: 8 + behavior: + scaleUp: + stabilizationWindowSeconds: 60 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 70 diff --git a/docs/examples/kubernetes/resource-profiles/wab-hpa.yaml b/docs/examples/kubernetes/resource-profiles/wab-hpa.yaml new file mode 100644 index 000000000..13bfa427b --- /dev/null +++ b/docs/examples/kubernetes/resource-profiles/wab-hpa.yaml @@ -0,0 +1,39 @@ +# Prerequisites: MySQL capacity for maxReplicas * WAB_DB_POOL_MAX and a shared +# gateway rate limit for authentication, SMS, faucet, deletion, and share APIs. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: wab +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: wab + minReplicas: 2 + maxReplicas: 10 + behavior: + scaleUp: + stabilizationWindowSeconds: 30 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 75 diff --git a/docs/examples/kubernetes/resource-profiles/wallet-storage-api-hpa.yaml b/docs/examples/kubernetes/resource-profiles/wallet-storage-api-hpa.yaml new file mode 100644 index 000000000..c601f62c0 --- /dev/null +++ b/docs/examples/kubernetes/resource-profiles/wallet-storage-api-hpa.yaml @@ -0,0 +1,39 @@ +# Target API-role pods only (WALLET_INFRA_ROLE=api). Keep exactly one separate +# WALLET_INFRA_ROLE=monitor pod outside this Deployment and HPA. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: wallet-storage-api +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: wallet-storage-api + minReplicas: 2 + maxReplicas: 8 + behavior: + scaleUp: + stabilizationWindowSeconds: 60 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 70 diff --git a/docs/infrastructure/chaintracks-server.md b/docs/infrastructure/chaintracks-server.md index 2d4dfb3c9..31c1dac7f 100644 --- a/docs/infrastructure/chaintracks-server.md +++ b/docs/infrastructure/chaintracks-server.md @@ -1,10 +1,10 @@ --- id: infra-chaintracks-server -title: "Chaintracks Server" +title: 'Chaintracks Server' kind: infra -version: "1.0.10" -last_updated: "2026-07-25" -last_verified: "2026-07-25" +version: '1.1.0' +last_updated: '2026-08-05' +last_verified: '2026-08-05' review_cadence_days: 30 status: stable tags: [chaintracks, block-headers, spv, merkle, infrastructure] @@ -22,15 +22,25 @@ The Chaintracks primitives and client interface are defined in `@bsv/wallet-tool - Maintains a chain of headers from genesis to the current tip via bulk + live ingestors - Exposes JSON v1 (legacy) and v2 (RESTful, with binary variants) APIs - Provides bulk header download in concatenated 80-byte format for SPV clients +- Tracks `main`, `test`, `stn`, `ttn`, and `tstn` with distinct validated genesis headers +- Uses credential-free Arcade/go-chaintracks bulk and SSE sources before the optional WhatsOnChain fallback ## Startup and Bootstrap On first start, Chaintracks must acquire all existing BSV block headers before serving SPV queries. The bootstrap sequence: -1. **Bundled files** — Repository ships with bulk header files. Used for initial ingest when no CDN URL is configured. -2. **CDN bulk ingest** — If `CHAINTRACKS_CDN_URL` is set (typically another running Chaintracks server), headers are fetched in 100,000-block batches. Fastest path. -3. **WhatsOnChain bulk ingester** — Fallback when bundled files and CDN are unavailable. -4. **Live tip sync** — Once bulk headers are loaded, switches to live mode via Teranode P2P or the WhatsOnChain live ingester. +1. **Retained/bundled files** — Previously validated local files remain the fastest and most independent bootstrap source. +2. **CDN bulk ingest** — `SOURCE_CDN_URL` supplies immutable bulk files. Set it to an empty string to disable this source without changing older deployments that rely on the default. +3. **Arcade/go-chaintracks** — The server fetches bounded binary header batches and follows the reconnecting tip SSE stream. Public defaults exist for mainnet, testnet, and TerraTestNet; STN and Terra Scaling TestNet require an operator endpoint. +4. **WhatsOnChain fallback** — Mainnet and testnet only. No key is required: anonymous requests are serialized below the documented three requests/second limit. A key can raise the allowance, but a rejected key is retried anonymously instead of making ChainTracks unavailable. + +Every remote batch still passes through ChainTracks' local serialization, hash, +continuity, and genesis checks before storage. When every provider is +temporarily unavailable, a synchronized process continues serving its +last-good checked height and headers and exposes degraded source state from +`getInfo`/`readyz`. + +Arcade is the HTTPS/SSE gateway used by browser, mobile, local, and service deployments and may itself be backed by Teranode P2P. This TypeScript server does not open a direct Teranode P2P session; adding one would require a separately reviewed server-only adapter and must not enter browser bundles. ## API @@ -38,34 +48,37 @@ Two API surfaces are mounted on the same port (default `3011`): ### v1 (JSON, legacy) -| Method | Path | Purpose | -|---|---|---| -| GET | `/getChain` | Network name (`main` or `test`) | -| GET | `/getInfo` | Service state: heights, storage backend, ingestors | -| GET | `/getPresentHeight` | Latest external blockchain height | -| GET | `/findChainTipHashHex` | Active chain tip hash | -| GET | `/findChainTipHeaderHex` | Active chain tip header | -| GET | `/findHeaderHexForHeight?height=N` | Header at height | -| GET | `/findHeaderHexForBlockHash?hash=H` | Header for hash (live storage) | -| GET | `/getHeaders?height=N&count=M` | Concatenated 80-byte hex header batch | -| GET | `/getFiatExchangeRates` | BSV fiat exchange rates | -| POST | `/addHeaderHex` | Submit a new block header for processing | +| Method | Path | Purpose | +| ------ | ----------------------------------- | ------------------------------------------------------ | +| GET | `/getChain` | Network name (`main`, `test`, `stn`, `ttn`, or `tstn`) | +| GET | `/getInfo` | Service state: heights, storage backend, ingestors | +| GET | `/getPresentHeight` | Latest external blockchain height | +| GET | `/findChainTipHashHex` | Active chain tip hash | +| GET | `/findChainTipHeaderHex` | Active chain tip header | +| GET | `/findHeaderHexForHeight?height=N` | Header at height | +| GET | `/findHeaderHexForBlockHash?hash=H` | Header for hash (live storage) | +| GET | `/getHeaders?height=N&count=M` | Concatenated 80-byte hex header batch | +| GET | `/getFiatExchangeRates` | BSV fiat exchange rates | +| POST | `/addHeaderHex` | Submit a new block header for processing | ### v2 (RESTful, JSON + binary) Mirrors the `go-chaintracks` v2 contract. All responses use the `{status, value}` / `{status, code, description}` envelope; binary variants return raw 80-byte headers with `X-Block-Height` / `X-Start-Height` / `X-Header-Count` headers. -| Method | Path | Purpose | -|---|---|---| -| GET | `/v2/network` | Network name | -| GET | `/v2/tip` | Chain tip header (JSON) | -| GET | `/v2/tip.bin` | Chain tip header (80-byte binary) | -| GET | `/v2/header/height/:height` | Header at height (JSON) | -| GET | `/v2/header/height/:height.bin` | Header at height (binary) | -| GET | `/v2/header/hash/:hash` | Header by hash (JSON) | -| GET | `/v2/header/hash/:hash.bin` | Header by hash (binary) | -| GET | `/v2/headers?height=N&count=M` | Header batch (binary, JSON envelope omitted) | -| GET | `/v2/headers.bin?height=N&count=M` | Header batch (binary) | +| Method | Path | Purpose | +| ------ | ---------------------------------- | -------------------------------------------- | +| GET | `/v2/network` | Network name | +| GET | `/v2/height` | Present height | +| GET | `/v2/tip` | Chain tip header (JSON) | +| GET | `/v2/tip/stream` | Reconnecting-compatible SSE tip stream | +| GET | `/v2/reorg/stream` | SSE reorganization stream | +| GET | `/v2/tip.bin` | Chain tip header (80-byte binary) | +| GET | `/v2/header/height/:height` | Header at height (JSON) | +| GET | `/v2/header/height/:height.bin` | Header at height (binary) | +| GET | `/v2/header/hash/:hash` | Header by hash (JSON) | +| GET | `/v2/header/hash/:hash.bin` | Header by hash (binary) | +| GET | `/v2/headers?height=N&count=M` | Header batch (binary, JSON envelope omitted) | +| GET | `/v2/headers.bin?height=N&count=M` | Header batch (binary) | The v2 surface is exercised by the [`sync.chaintracks-v2-http`](../conformance/index.md) conformance vectors so cross-language implementations (`go-chaintracks`, future Rust/Python ports) can be validated against the same contract. @@ -73,12 +86,28 @@ The v2 surface is exercised by the [`sync.chaintracks-v2-http`](../conformance/i ```bash PORT=3011 # HTTP listen port -CHAIN=main # main | test -CHAINTRACKS_CDN_URL=https://chaintracks-us-1.bsvb.tech # CDN bootstrap source -WHATS_ON_CHAIN_LIVE=true # Use WhatsOnChain live ingester instead of Teranode +CHAIN=main # main | test | stn | ttn | tstn +SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ +CHAINTRACKS_UPSTREAM_URL= # optional override; "disabled" disables +CHAINTRACKS_UPSTREAM_API_PREFIX= # inferred as /chaintracks/v2 by default +CHAINTRACKS_UPSTREAM_MAX_HEADERS=1000 +CHAINTRACKS_DISABLE_WHATSONCHAIN=false # main/test fallback only +WHATSONCHAIN_API_KEY= # optional, never required +STN_ARCADE_URL= # optional STN v2 host +STN_CHAINTRACKS_URL= # optional STN v2 ChainTracks URL +TSTN_ARCADE_URL= # optional TSTN v2 host +TSTN_CHAINTRACKS_URL= # optional TSTN v2 ChainTracks URL +ROUTING_PREFIX= # optional mount prefix; /healthz stays at root ``` -Teranode P2P live ingest requires bootstrap peer configuration. +For `stn` or `tstn`, set `CHAINTRACKS_UPSTREAM_URL` (or the corresponding +`STN_ARCADE_URL`, `STN_CHAINTRACKS_URL`, `TSTN_ARCADE_URL`, or +`TSTN_CHAINTRACKS_URL`) to an operator-controlled Arcade or go-chaintracks v2 +service. The server fails closed rather than aliasing either network to testnet. + +`GET /healthz` is the process liveness endpoint. `GET /readyz` (under +`ROUTING_PREFIX` when configured) verifies that ChainTracks is listening and +can report a locally or remotely sourced height; it also includes source health. Public browser access is enabled by default. Use `CHAINTRACKS_CORS_MODE=allowlist` and diff --git a/docs/infrastructure/message-box-server.md b/docs/infrastructure/message-box-server.md index fac115f68..74a180c48 100644 --- a/docs/infrastructure/message-box-server.md +++ b/docs/infrastructure/message-box-server.md @@ -40,7 +40,7 @@ Clients connect with identity-based authentication, send and receive messages th | Method | Path | Purpose | | ------ | -------------------- | ------------------------------------------------------------------ | | POST | /sendMessage | Send encrypted message to recipient (authenticated) | -| POST | /listMessages | List all unacknowledged messages in box (authenticated) | +| POST | /listMessages | Page unacknowledged messages with `limit` and `offset`/`skip` | | POST | /acknowledgeMessage | Mark messages as read/delete them (authenticated) | | POST | /registerDevice | Register a push-notification device for the authenticated identity | | GET | /devices | List the authenticated identity's devices with redacted tokens | @@ -52,6 +52,11 @@ Clients connect with identity-based authentication, send and receive messages th | GET | /ready | Public database readiness | | GET | /docs, /openapi.json | Public API documentation | +The omitted page size is 1,000 in the standard profile and is operator +configurable. Responses include `limit`, `offset`, `nextOffset`, and `hasMore`. +See [Service Resource Profiles](../reference/service-resource-profiles.md) for +all limits, shared state, BRC-105 pricing, memory evidence, and scaling guidance. + ## WebSocket endpoints - **ws://host:8080** – Authenticated WebSocket server using @bsv/authsocket diff --git a/docs/packages/messaging/message-box-client.md b/docs/packages/messaging/message-box-client.md index 0a03d5308..9496abb8d 100644 --- a/docs/packages/messaging/message-box-client.md +++ b/docs/packages/messaging/message-box-client.md @@ -3,10 +3,10 @@ id: pkg-message-box-client title: '@bsv/message-box-client' kind: package domain: messaging -version: '2.2.6' +version: '2.3.0' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-07-30' -last_verified: '2026-07-30' +last_updated: '2026-08-04' +last_verified: '2026-08-04' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/message-box-client' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/message-box-client' diff --git a/docs/packages/middleware/auth-express-middleware.md b/docs/packages/middleware/auth-express-middleware.md index f0042dd1b..4f8d5f0e4 100644 --- a/docs/packages/middleware/auth-express-middleware.md +++ b/docs/packages/middleware/auth-express-middleware.md @@ -3,10 +3,10 @@ id: pkg-auth-express-middleware title: '@bsv/auth-express-middleware' kind: package domain: middleware -version: '2.1.7' +version: '2.2.0' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-07-31' -last_verified: '2026-07-31' +last_updated: '2026-08-04' +last_verified: '2026-08-04' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/auth-express-middleware' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware' @@ -67,15 +67,20 @@ app.use( logLevel: 'error', transportLimits: { requestTimeoutMs: 30_000, - maxPendingRequests: 1_000 + maxPendingRequests: 1_000, + maxResponseBytes: 8 * 1024 * 1024 } }) ) ``` `transportLimits` bounds pending handshakes, verification listeners, -certificate waits, and response-signing state. Malformed requests are rejected -before state allocation. At capacity, the middleware fails closed with `503`. +certificate waits, and response-signing state. `maxResponseBytes` also bounds +files passed to `res.sendFile`; oversized application responses fail closed +with a signed `413`. The default is 8 MiB. Operators may set it to `-1` only +when the embedding service enforces an equivalent response budget. Malformed +requests are rejected before state allocation. At capacity, the middleware +fails closed with `503`. The exact `/.well-known/auth` endpoint remains public because it establishes the session. Similar path prefixes receive normal auth treatment. @@ -138,7 +143,8 @@ CORS. CSP is primarily a document policy and is not a substitute for API CORS. - Use HTTPS; mutual authentication does not encrypt all HTTP data. - Parse bodies before auth so signed and routed values match. - Install one auth wrapper per request path. -- Keep finite timeouts/capacity and alert on `408` and `503`. +- Keep finite timeouts/response sizes/capacity and alert on `408`, `413`, and + `503`. - Keep authentication separate from application authorization. - Do not log raw headers, signatures, certificates, bodies, or wallet objects. - Public errors are stable and omit internal exception text. diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index 6cb4c6b8b..39b635d19 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -4,9 +4,9 @@ title: '@bsv/overlay-express' kind: package domain: overlays npm: '@bsv/overlay-express' -version: '2.4.9' -last_updated: '2026-07-30' -last_verified: '2026-07-30' +version: '2.5.0' +last_updated: '2026-08-04' +last_verified: '2026-08-04' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express' status: stable diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 2cc6f01e7..28787ed16 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -4,9 +4,9 @@ title: '@bsv/overlay' kind: package domain: overlays npm: '@bsv/overlay' -version: '2.2.7' -last_updated: '2026-07-30' -last_verified: '2026-07-30' +version: '2.3.0' +last_updated: '2026-08-04' +last_verified: '2026-08-04' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay' status: stable diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index 3ce9b4887..135b7b7fc 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -3,9 +3,9 @@ id: pkg-wallet-toolbox-client title: '@bsv/wallet-toolbox-client' kind: package domain: wallet -version: '2.5.0' -last_updated: '2026-08-04' -last_verified: '2026-08-04' +version: '2.6.0' +last_updated: '2026-08-05' +last_verified: '2026-08-05' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/wallet-toolbox-client' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client' @@ -24,6 +24,11 @@ Browser authentication accepts one verified matching UMP token as an existing account. When no token verifies, one clean empty overlay response establishes a new account even if other hosts fail or return malformed records. +The browser build includes the fetch-based, credential-free ChainTracks v2 +client and reconnecting SSE adapter without Node-only modules. Public defaults +cover mainnet, testnet, and TerraTestNet; STN/TSTN use an injected or configured +endpoint. + ## Install ```bash diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index 24fde933a..94b6e2c76 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -3,9 +3,9 @@ id: pkg-wallet-toolbox-mobile title: '@bsv/wallet-toolbox-mobile' kind: package domain: wallet -version: '2.5.0' -last_updated: '2026-08-04' -last_verified: '2026-08-04' +version: '2.6.0' +last_updated: '2026-08-05' +last_verified: '2026-08-05' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/wallet-toolbox-mobile' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile' @@ -24,6 +24,11 @@ Mobile authentication accepts one verified matching UMP token as an existing account. When no token verifies, one clean empty overlay response establishes a new account even if other hosts fail or return malformed records. +The mobile build includes the fetch-based, credential-free ChainTracks v2 +client and reconnecting SSE adapter without Node-only modules. Public defaults +cover mainnet, testnet, and TerraTestNet; STN/TSTN use an injected or configured +endpoint. + ## Install ```bash diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index b3fae2f50..e5ee0f2cf 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -4,9 +4,9 @@ title: '@bsv/wallet-toolbox' kind: package domain: wallet npm: '@bsv/wallet-toolbox' -version: '2.5.0' -last_updated: '2026-08-04' -last_verified: '2026-08-04' +version: '2.6.0' +last_updated: '2026-08-05' +last_verified: '2026-08-05' review_cadence_days: 30 status: stable tags: ['wallet', 'brc100'] @@ -29,6 +29,14 @@ account even if other hosts fail or return malformed records. Multiple distinct verified tokens and lookups with no usable response remain errors; WAB existing-account continuity still prevents replacement-wallet onboarding. +ChainTracks defaults to credential-free Arcade/go-chaintracks v2 HTTP and SSE +on mainnet, testnet, and TerraTestNet. STN and Terra Scaling TestNet require an +explicit operator endpoint. Remote header batches pass local serialization, +hash, continuity, and genesis checks; source failures fall through in priority +order; and synchronized trackers keep serving last-good local data. +WhatsOnChain is an optional, rate-limited mainnet/testnet fallback; no key is +required. + ## Install ```bash diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 0e0259b60..fcaccc736 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -3,8 +3,8 @@ id: package-api-migrations title: 'Package API, Declarations, and Migration Ledger' kind: reference version: '1.0.0' -last_updated: '2026-08-03' -last_verified: '2026-08-03' +last_updated: '2026-08-04' +last_verified: '2026-08-04' review_cadence_days: 30 status: stable tags: [reference, packages, api, declarations, migrations, release-notes] @@ -23,39 +23,39 @@ and clean-consumer tests remain the executable type authority. ## Current release boundary -| Package | npm baseline | Source | Candidate | API | Migration | -| --------------------------------- | ------------ | ------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@bsv/402-pay` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. | -| `@bsv/air-gap` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | -| `@bsv/amountinator` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | -| `@bsv/auth` | `0.1.1` | `0.1.3` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | -| `@bsv/auth-express-middleware` | `2.1.2` | `2.1.7` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No consumer migration is required; existing public CORS defaults and middleware APIs are retained, authentication callback failures now produce a controlled HTTP error, and Express 4 and 5 applications use their own peer-provided Express installation. | -| `@bsv/authsocket` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/messaging/authsocket.md) | Valid traffic and the wire contract are unchanged. A socket is now disconnected when its authentication processing exceeds the concurrency limit or a callback fails; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | -| `@bsv/authsocket-client` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/messaging/authsocket-client.md) | Valid traffic and supported imports are unchanged. The client now disconnects from a server that causes authentication failure or exceeds the concurrency limit; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | -| `@bsv/btms` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms.md) | No consumer migration is required; token and lookup wire contracts are unchanged. | -| `@bsv/btms-permission-module` | `1.1.1` | `1.1.3` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | -| `@bsv/did` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | -| `@bsv/did-client` | `1.2.1` | `1.2.3` | patch | [API and usage](../packages/helpers/did-client.md) | No consumer migration is required; DID client APIs, encodings, and supported imports are unchanged. | -| `@bsv/fund-wallet` | `1.4.1` | `1.4.3` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. | -| `@bsv/gasp` | `1.3.1` | `1.3.5` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. | -| `@bsv/message-box-client` | `2.2.2` | `2.2.6` | patch | [API and usage](../packages/messaging/message-box-client.md) | No consumer migration is required; Message Box protocol and client entry points are unchanged. | -| `@bsv/overlay` | `2.2.1` | `2.2.7` | patch | [API and usage](../packages/overlays/overlay.md) | No consumer migration is required; existing imports, submission results, notification contracts, storage order, and network behavior remain unchanged. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/overlays/overlay-discovery-services.md) | No consumer migration is required; discovery records and public network behavior are unchanged. | -| `@bsv/overlay-express` | `2.4.2` | `2.4.9` | patch | [API and usage](../packages/overlays/overlay-express.md) | No consumer migration is required; wildcard credential-free public access remains the default and runtimes may opt into the new close method. | -| `@bsv/overlay-topics` | `1.6.1` | `1.6.8` | patch | [API and usage](../packages/overlays/overlay-topics.md) | No consumer migration is required; topic IDs, lookup contracts, and persisted formats are unchanged. | -| `@bsv/paymail` | `2.4.2` | `2.4.6` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. | -| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. | -| `@bsv/sdk` | `2.2.18` | `2.3.0` | minor | [API and usage](../packages/sdk/bsv-sdk.md) | No consumer migration is required. The new APIs are additive; BEEF ordering and bytes, proof validity, signatures, synchronous signing, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | -| `@bsv/simple` | `0.4.1` | `0.4.8` | patch | [API and usage](../packages/helpers/simple.md) | No consumer migration is required; the browser and server entry points remain compatible. | -| `@bsv/templates` | `1.9.1` | `1.9.6` | patch | [API and usage](../packages/helpers/templates.md) | No consumer migration is required; template APIs, supported imports, and generated script semantics are unchanged. | -| `@bsv/teranode-listener` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. | -| `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | -| `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | -| `@bsv/wallet-relay` | `0.2.2` | `0.3.4` | minor | [API and usage](../packages/wallet/wallet-relay.md) | QRPairingCode now renders a native button and accepts button wrapper attributes. Existing className, style, data, and ARIA props continue to work; update div-specific wrapper selectors or explicitly typed div event handlers. Express integrations now use the host application's matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.4.22` | `2.5.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No API or persistence migration is required. Storage-provider additions are backward-compatible with fallbacks and existing databases use the normal migration path. Applications may now enter new-user flow when at least one overlay host returns a clean empty result despite malformed or unavailable peers; one verified token still establishes an existing account, multiple unresolved verified tokens remain an error, and WAB existing-account continuity still blocks replacement-wallet onboarding. Wallet results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are otherwise unchanged. | -| `@bsv/wallet-toolbox-client` | `2.4.22` | `2.5.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No API or persistence migration is required. Browser wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox; older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. | -| `@bsv/wallet-toolbox-mobile` | `2.4.22` | `2.5.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No API or persistence migration is required. Mobile wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox; older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. | -| `create-bsv-app` | `1.0.2` | `1.0.4` | patch | [API and usage](../packages/helpers/create-bsv-app.md) | No consumer migration is required; generated application structure and CLI behavior are unchanged. | +| Package | npm baseline | Source | Candidate | API | Migration | +| --------------------------------- | ------------ | ------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `@bsv/402-pay` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. | +| `@bsv/air-gap` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | +| `@bsv/amountinator` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | +| `@bsv/auth` | `0.1.1` | `0.1.3` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | +| `@bsv/auth-express-middleware` | `2.1.2` | `2.2.0` | minor | [API and usage](../packages/middleware/auth-express-middleware.md) | Existing middleware construction remains compatible. Operators may set transportLimits.maxResponseBytes to a positive byte count or -1 when the embedding service enforces an equivalent response budget. | +| `@bsv/authsocket` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/messaging/authsocket.md) | Valid traffic and the wire contract are unchanged. A socket is now disconnected when its authentication processing exceeds the concurrency limit or a callback fails; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | +| `@bsv/authsocket-client` | `2.1.1` | `2.1.4` | patch | [API and usage](../packages/messaging/authsocket-client.md) | Valid traffic and supported imports are unchanged. The client now disconnects from a server that causes authentication failure or exceeds the concurrency limit; use onError for diagnostics and maxPendingAuthMessages to tune the default limit of 32. | +| `@bsv/btms` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms.md) | No consumer migration is required; token and lookup wire contracts are unchanged. | +| `@bsv/btms-permission-module` | `1.1.1` | `1.1.3` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | +| `@bsv/did` | `0.2.1` | `0.2.4` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | +| `@bsv/did-client` | `1.2.1` | `1.2.3` | patch | [API and usage](../packages/helpers/did-client.md) | No consumer migration is required; DID client APIs, encodings, and supported imports are unchanged. | +| `@bsv/fund-wallet` | `1.4.1` | `1.4.3` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. | +| `@bsv/gasp` | `1.3.1` | `1.3.5` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. | +| `@bsv/message-box-client` | `2.2.2` | `2.3.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | Existing listMessages and listMessagesLite calls continue to fetch all available messages. Applications that need an aggregate memory ceiling should set limit and/or maxPages; no additional BRC-105 approval callback is required because AuthFetch uses wallet permissions. | +| `@bsv/overlay` | `2.2.1` | `2.3.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine constructor calls remain valid and default to 1,000 lookup formulas. Pass -1 as the final maxLookupResults argument only when a custom lookup service and deployment enforce an equivalent bound. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/overlays/overlay-discovery-services.md) | No consumer migration is required; discovery records and public network behavior are unchanged. | +| `@bsv/overlay-express` | `2.4.2` | `2.5.0` | minor | [API and usage](../packages/overlays/overlay-express.md) | The standard profile is selected by default. Existing configuration methods remain compatible; use OVERLAY_RESOURCE_PROFILE or granular OVERLAY_* limits, and ensure custom lookup services bound their own database queries before returning formulas. | +| `@bsv/overlay-topics` | `1.6.1` | `1.6.8` | patch | [API and usage](../packages/overlays/overlay-topics.md) | No consumer migration is required; topic IDs, lookup contracts, and persisted formats are unchanged. | +| `@bsv/paymail` | `2.4.2` | `2.4.6` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. | +| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. | +| `@bsv/sdk` | `2.2.18` | `2.3.0` | minor | [API and usage](../packages/sdk/bsv-sdk.md) | No consumer migration is required. The new APIs are additive; BEEF ordering and bytes, proof validity, signatures, synchronous signing, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | +| `@bsv/simple` | `0.4.1` | `0.4.8` | patch | [API and usage](../packages/helpers/simple.md) | No consumer migration is required; the browser and server entry points remain compatible. | +| `@bsv/templates` | `1.9.1` | `1.9.6` | patch | [API and usage](../packages/helpers/templates.md) | No consumer migration is required; template APIs, supported imports, and generated script semantics are unchanged. | +| `@bsv/teranode-listener` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. | +| `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | +| `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | +| `@bsv/wallet-relay` | `0.2.2` | `0.3.4` | minor | [API and usage](../packages/wallet/wallet-relay.md) | QRPairingCode now renders a native button and accepts button wrapper attributes. Existing className, style, data, and ARIA props continue to work; update div-specific wrapper selectors or explicitly typed div event handlers. Express integrations now use the host application's matching Express runtime and type graph. | +| `@bsv/wallet-toolbox` | `2.5.0` | `2.6.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | List/find RPC calls that omit a limit now receive the profile default (1,000 in standard), and larger caller limits are rejected above the operator maximum. Existing calls and storage data remain compatible; migrations add auth_sessions, payment_replays, and createAction indexes automatically, and storage-provider additions retain backward-compatible fallbacks. Applications may now enter new-user flow when at least one overlay host returns a clean empty result despite malformed or unavailable peers; one verified token still establishes an existing account, multiple unresolved verified tokens remain an error, and WAB existing-account continuity still blocks replacement-wallet onboarding. The default ChainTracks client for mainnet, testnet, and TerraTestNet now uses each public Arcade v2 endpoint without a credential; explicit ChaintracksClientApi injection and legacy v1 URLs remain supported. STN and Terra Scaling TestNet require an explicit STN_CHAINTRACKS_URL/TSTN_CHAINTRACKS_URL or corresponding Arcade URL. A WhatsOnChain key is optional, anonymous fallback stays below the documented public rate, and ChainTracks header/info requests retry anonymously when a configured key is rejected. Browser/mobile CORS expectations, wallet results, BRC-103/104, AuthFetch, JSON-RPC, and wallet wire behavior are otherwise unchanged. | +| `@bsv/wallet-toolbox-client` | `2.5.0` | `2.6.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No browser consumer migration is required. Remote storage callers may provide explicit list limits when they need a value other than the operator profile default, and browser wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox. Mainnet, testnet, and TerraTestNet ChainTracks defaults now use public credential-free Arcade v2 endpoints; explicit clients and legacy v1 URLs remain compatible, while STN and Terra Scaling TestNet require an explicit endpoint. Older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and CORS, BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage wire behavior remain compatible. | +| `@bsv/wallet-toolbox-mobile` | `2.5.0` | `2.6.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No mobile consumer migration is required. Remote storage callers may provide explicit list limits when they need a value other than the operator profile default, and mobile wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox. Mainnet, testnet, and TerraTestNet ChainTracks defaults now use public credential-free Arcade v2 endpoints; explicit clients and legacy v1 URLs remain compatible, while STN and Terra Scaling TestNet require an explicit endpoint. Older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage wire behavior remain compatible. | +| `create-bsv-app` | `1.0.2` | `1.0.4` | patch | [API and usage](../packages/helpers/create-bsv-app.md) | No consumer migration is required; generated application structure and CLI behavior are unchanged. | `none` means the source manifest matches the recorded npm baseline. Any other value is an unpublished candidate. Publication, tags, releases, registry @@ -114,8 +114,8 @@ explicitly authorized operations. - Package documentation: [docs/packages/middleware/auth-express-middleware.md](../packages/middleware/auth-express-middleware.md) - Source: [packages/middleware/auth-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware) -- Release note: Standardizes package quality and strengthens authenticated Express session, edge-policy, and error-handling boundaries, including containment of both synchronous and asynchronous BRC-103 callback failures, and restores one shared Express 4/5 runtime and type graph for consumers. -- Migration: No consumer migration is required; existing public CORS defaults and middleware APIs are retained, authentication callback failures now produce a controlled HTTP error, and Express 4 and 5 applications use their own peer-provided Express installation. +- Release note: Bounds BRC-104 application-response capture before authenticated signing, with an 8 MiB default, a configurable transport limit, stable 413 errors, and boundary regression coverage. +- Migration: Existing middleware construction remains compatible. Operators may set transportLimits.maxResponseBytes to a positive byte count or -1 when the embedding service enforces an equivalent response budget. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ------------------------------------ | ---------------------------------------- | @@ -222,8 +222,8 @@ CLI entry points: `{"fund-metanet":"./dist/index.mjs"}`. - Package documentation: [docs/packages/messaging/message-box-client.md](../packages/messaging/message-box-client.md) - Source: [packages/messaging/message-box-client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/message-box-client) -- Release note: Adds strict package contracts and hardens PeerPay parsing, proof handling, cancellation, classification, and acknowledgement flows. -- Migration: No consumer migration is required; Message Box protocol and client entry points are unchanged. +- Release note: Follows bounded Message Box server pagination while preserving historical fetch-all behavior and adds offset/skip, total-limit, page-size, and optional page-ceiling controls. +- Migration: Existing listMessages and listMessagesLite calls continue to fetch all available messages. Applications that need an aggregate memory ceiling should set limit and/or maxPages; no additional BRC-105 approval callback is required because AuthFetch uses wallet permissions. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ----------------------------------- | --------------------------------------- | @@ -234,8 +234,8 @@ CLI entry points: `{"fund-metanet":"./dist/index.mjs"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Repairs the documented storage export and decomposes Engine submission validation, broadcast, storage, notification, and propagation orchestration while preserving execution order and behavior. -- Migration: No consumer migration is required; existing imports, submission results, notification contracts, storage order, and network behavior remain unchanged. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration so a compact remote query cannot amplify into unbounded retained work. +- Migration: Existing Engine constructor calls remain valid and default to 1,000 lookup formulas. Pass -1 as the final maxLookupResults argument only when a custom lookup service and deployment enforce an equivalent bound. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | @@ -261,8 +261,8 @@ CLI entry points: `{"fund-metanet":"./dist/index.mjs"}`. - Package documentation: [docs/packages/overlays/overlay-express.md](../packages/overlays/overlay-express.md) - Source: [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) -- Release note: Adds strict package and edge-policy contracts, restores declaration-safe exports, adds idempotent shutdown, and hardens provider-chain failure handling and synchronization configuration. -- Migration: No consumer migration is required; wildcard credential-free public access remains the default and runtimes may opt into the new close method. +- Release note: Adds small, standard, and high-throughput resource profiles; bounded lookup, BASM, admin, connection, body, and response work; and streaming janitor scans with independently capped report retention. +- Migration: The standard profile is selected by default. Existing configuration methods remain compatible; use OVERLAY_RESOURCE_PROFILE or granular OVERLAY_* limits, and ensure custom lookup services bound their own database queries before returning formulas. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -477,8 +477,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Makes the successful fragmented createAction path atomic and set-based, overlaps batched proof reads with persistence, batch-validates compound proofs and canonical P2PKH signatures, shares BRC-42 derivation work, removes unused commit reads, bulk-inserts outputs, coalesces authenticated timestamp-only Knex session touches, adds timings for every remaining material phase, and makes UMP account lookup resilient to partial overlay failure. -- Migration: No API or persistence migration is required. Storage-provider additions are backward-compatible with fallbacks and existing databases use the normal migration path. Applications may now enter new-user flow when at least one overlay host returns a clean empty result despite malformed or unavailable peers; one verified token still establishes an existing account, multiple unresolved verified tokens remain an error, and WAB existing-account continuity still blocks replacement-wallet onboarding. Wallet results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are otherwise unchanged. +- Release note: Adds resource-profile-aware Wallet Storage RPC pagination, array and response budgets, bounded authenticated response capture, durable payment replay and session schemas, and double-slash compatibility; makes fragmented createAction atomic and set-based with batched proof reads, validation, output writes, shared derivation work, coalesced session touches, and complete privacy-safe phase timings; makes UMP account lookup resilient to partial overlay failure; and refreshes ChainTracks with credential-free Arcade/go-chaintracks bulk and SSE sources, five-network genesis correctness, prioritized failover, local last-good operation, source health, browser-safe reconnects, and a rate-limited keyless WhatsOnChain fallback. +- Migration: List/find RPC calls that omit a limit now receive the profile default (1,000 in standard), and larger caller limits are rejected above the operator maximum. Existing calls and storage data remain compatible; migrations add auth_sessions, payment_replays, and createAction indexes automatically, and storage-provider additions retain backward-compatible fallbacks. Applications may now enter new-user flow when at least one overlay host returns a clean empty result despite malformed or unavailable peers; one verified token still establishes an existing account, multiple unresolved verified tokens remain an error, and WAB existing-account continuity still blocks replacement-wallet onboarding. The default ChainTracks client for mainnet, testnet, and TerraTestNet now uses each public Arcade v2 endpoint without a credential; explicit ChaintracksClientApi injection and legacy v1 URLs remain supported. STN and Terra Scaling TestNet require an explicit STN_CHAINTRACKS_URL/TSTN_CHAINTRACKS_URL or corresponding Arcade URL. A WhatsOnChain key is optional, anonymous fallback stays below the documented public rate, and ChainTracks header/info requests retry anonymously when a configured key is rejected. Browser/mobile CORS expectations, wallet results, BRC-103/104, AuthFetch, JSON-RPC, and wallet wire behavior are otherwise unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -491,8 +491,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Carries the lockstep browser build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, and resilient UMP account lookup. -- Migration: No API or persistence migration is required. Browser wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox; older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. +- Release note: Carries the lockstep browser build with the bounded remote-storage contract, batched proof assembly, linear funding and signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, resilient UMP account lookup, and the browser-safe credential-free ChainTracks v2 client with reconnecting SSE and five-network header support. +- Migration: No browser consumer migration is required. Remote storage callers may provide explicit list limits when they need a value other than the operator profile default, and browser wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox. Mainnet, testnet, and TerraTestNet ChainTracks defaults now use public credential-free Arcade v2 endpoints; explicit clients and legacy v1 URLs remain compatible, while STN and Terra Scaling TestNet require an explicit endpoint. Older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and CORS, BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage wire behavior remain compatible. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -503,8 +503,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Carries the lockstep mobile build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, and resilient UMP account lookup. -- Migration: No API or persistence migration is required. Mobile wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox; older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. +- Release note: Carries the lockstep mobile build with the bounded remote-storage contract, batched proof assembly, linear funding and signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, resilient UMP account lookup, and the mobile-safe credential-free ChainTracks v2 client with reconnecting SSE and five-network header support. +- Migration: No mobile consumer migration is required. Remote storage callers may provide explicit list limits when they need a value other than the operator profile default, and mobile wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox. Mainnet, testnet, and TerraTestNet ChainTracks defaults now use public credential-free Arcade v2 endpoints; explicit clients and legacy v1 URLs remain compatible, while STN and Terra Scaling TestNet require an explicit endpoint. Older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage wire behavior remain compatible. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/service-operations.md b/docs/reference/service-operations.md index 27d42365e..49af73abc 100644 --- a/docs/reference/service-operations.md +++ b/docs/reference/service-operations.md @@ -3,8 +3,8 @@ id: service-operations title: 'Service Operations Contract' kind: reference version: '2.0.0' -last_updated: '2026-07-29' -last_verified: '2026-07-29' +last_updated: '2026-08-05' +last_verified: '2026-08-05' review_cadence_days: 30 status: stable tags: [reference, infrastructure, operations, observability, slo, recovery] @@ -26,15 +26,15 @@ and CSP remains an independent document/UI policy rather than API authorization. ## Runtime endpoints and lifecycle -| Service | Port contract | Liveness | Readiness | Lifecycle | Operations | -| -------------------------- | -------------------------------------------------------- | -------------- | --------------- | ----------- | -------------------------------------------------------------------------------------------------------------- | -| `chaintracks-server` | PORT (default 3011; CDN is port + 1) | `/getInfo` | `/getInfo` | implemented | [guide](../infrastructure/chaintracks-server.md) | -| `message-box-server` | PORT, then HTTP_PORT (default 8080) | `/health` | `/ready` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/message-box-server/DEPLOYING.md) | -| `overlay-server` | 8080 | `/health/live` | `/health/ready` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/overlay-server/deploy/README.md) | -| `uhrp-server-basic` | HTTP_PORT (default 8080) | `/health` | `/ready` | implemented | [guide](../infrastructure/uhrp-server-basic.md) | -| `uhrp-server-cloud-bucket` | HTTP_PORT (default 8080) | `/health` | `/ready` | implemented | [guide](../infrastructure/uhrp-server-cloud-bucket.md) | -| `wab` | PORT (default 8080) | `/info` | `/info` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wab/deploy/README.md) | -| `wallet-infra` | HTTP_PORT (default 8081; samples set 8080 without nginx) | `/` | `/` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wallet-infra/guides/kube_samples/README.md) | +| Service | Port contract | Liveness | Readiness | Lifecycle | Operations | +| -------------------------- | -------------------------------------------------------- | ---------- | --------------- | ----------- | -------------------------------------------------------------------------------------------------------------- | +| `chaintracks-server` | PORT (default 3011; CDN is port + 1) | `/healthz` | `/readyz` | implemented | [guide](../infrastructure/chaintracks-server.md) | +| `message-box-server` | PORT, then HTTP_PORT (default 8080) | `/healthz` | `/ready` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/message-box-server/DEPLOYING.md) | +| `overlay-server` | 8080 | `/healthz` | `/health/ready` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/overlay-server/deploy/README.md) | +| `uhrp-server-basic` | HTTP_PORT (default 8080) | `/healthz` | `/ready` | implemented | [guide](../infrastructure/uhrp-server-basic.md) | +| `uhrp-server-cloud-bucket` | HTTP_PORT (default 8080) | `/healthz` | `/ready` | implemented | [guide](../infrastructure/uhrp-server-cloud-bucket.md) | +| `wab` | PORT (default 8080) | `/healthz` | `/info` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wab/deploy/README.md) | +| `wallet-infra` | HTTP_PORT (default 8081; samples set 8080 without nginx) | `/healthz` | `/` | implemented | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wallet-infra/guides/kube_samples/README.md) | Health endpoints are public and non-sensitive. They do not replace protocol authentication, administrative authorization, rate limits, or dependency-aware @@ -104,7 +104,7 @@ Incident handling follows this evidence-preserving sequence: ### chaintracks-server - Configuration: required `CHAIN`; optional - `BULK_HEADERS_PATH`, `CDN_HOST_URL`, `ENABLE_BULK_HEADERS_CDN`, `PORT`, `SOURCE_CDN_URL`, `WHATSONCHAIN_API_KEY`; secret-bearing + `BULK_HEADERS_PATH`, `CDN_HOST_URL`, `CHAINTRACKS_DISABLE_WHATSONCHAIN`, `CHAINTRACKS_UPSTREAM_API_PREFIX`, `CHAINTRACKS_UPSTREAM_MAX_HEADERS`, `CHAINTRACKS_UPSTREAM_URL`, `ENABLE_BULK_HEADERS_CDN`, `PORT`, `ROUTING_PREFIX`, `SOURCE_CDN_URL`, `STN_ARCADE_URL`, `STN_CHAINTRACKS_URL`, `TSTN_ARCADE_URL`, `TSTN_CHAINTRACKS_URL`, `WHATSONCHAIN_API_KEY`; secret-bearing `OTEL_EXPORTER_OTLP_HEADERS`, `WHATSONCHAIN_API_KEY`. - Telemetry: CJS bootstrap `src/telemetry.ts`, logger @@ -116,7 +116,7 @@ Incident handling follows this evidence-preserving sequence: - ingest and persist a verified header - Alerts: - header tip age or height stops advancing -- bulk-header export or upstream retrieval repeatedly fails +- all configured bulk/live sources are degraded or upstream retrieval repeatedly fails - API or CDN saturation exceeds its independent concurrency budget - State: Bulk-header files under BULK_HEADERS_PATH; upstream headers are reproducible. - Migration/startup: No schema migration. Validate the retained header corpus before rollout. diff --git a/docs/reference/service-resource-profiles.md b/docs/reference/service-resource-profiles.md new file mode 100644 index 000000000..0e93d1361 --- /dev/null +++ b/docs/reference/service-resource-profiles.md @@ -0,0 +1,250 @@ +--- +id: service-resource-profiles +title: 'Service Resource Profiles, Scaling, and Message Box Economics' +kind: reference +version: '1.0.0' +last_updated: '2026-08-04' +last_verified: '2026-08-04' +review_cadence_days: 30 +status: stable +tags: [reference, infrastructure, resource-safety, scaling, message-box, brc-105] +--- + +# Service Resource Profiles, Scaling, and Message Box Economics + +The official TS Stack service images reject remotely controlled work before it +can grow without a configured bound. The default `standard` profile targets a +pod with at least 1 GiB of memory. Operators can tune every cardinality, body, +response, connection, concurrency, retained-state, and maintenance ceiling at +runtime without rebuilding an image. + +These controls reduce known OOM paths; they do not make an “OOM-proof” promise. +Database drivers, telemetry, native libraries, custom lookup services, and the +kernel still require headroom and production load testing. + +## Configuration precedence and unlimited mode + +Set `RESOURCE_PROFILE` globally or `_RESOURCE_PROFILE` for one service. +Valid profiles are `small`, `standard`, and `high-throughput`; the prefixed +setting wins. A granular environment value wins over its profile. + +Resource ceilings accept a positive safe integer. Set a ceiling to `-1` or +`unlimited` only to make an explicit operator opt-out. Omission never means +unlimited. Finite protocol timeouts and database pool sizes remain positive +integers because disabling them can strand sockets or database waiters. + +The common service prefixes are `CHAINTRACKS`, `MESSAGE_BOX`, `OVERLAY`, +`UHRP`, `WAB`, and `WALLET_STORAGE`. + +| Common control | Meaning | +| --- | --- | +| `_MAX_BODY_BYTES` | Default materialized request body ceiling. JSON or binary routes may use a more specific prefix such as `UHRP_JSON` or `WALLET_STORAGE_BINARY`. | +| `_MAX_RESPONSE_BYTES` | Materialized response ceiling; a response above it receives `413 ERR_RESPONSE_TOO_LARGE`. | +| `_MAX_CONCURRENT_REQUESTS` | In-flight application requests per process; saturation receives `503 ERR_SERVER_BUSY`. | +| `_MAX_CONNECTIONS` | Open TCP/WebSocket connections per process. | +| `_REQUEST_TIMEOUT_MS` | Complete-request timeout. | +| `_HEADERS_TIMEOUT_MS` | Header receive timeout. | +| `_KEEP_ALIVE_TIMEOUT_MS` | Idle keep-alive timeout. | +| `_SOCKET_TIMEOUT_MS` | Socket inactivity timeout. | +| `_MAX_REQUESTS_PER_SOCKET` | Requests accepted before connection recycling. | +| `_MAX` / `_WINDOW_MS` | Route-class rate limit and window. Rate maxima accept `-1`/`unlimited`; windows remain finite. | + +Every API role exposes `/healthz` in addition to its existing health/readiness +contract. Two or more initial slashes are normalized for compatibility, so +`//healthz` continues to work without changing interior path semantics. + +## Profile defaults + +The table shows the dominant list/range maximum, response ceiling, and +per-process request concurrency. Route-specific defaults follow in the next +section. + +| Service | Small | Standard | High-throughput | +| --- | --- | --- | --- | +| Chaintracks | 500 headers, 1 MiB, 32 | 1,000 headers, 4 MiB, 64 | 5,000 headers, 32 MiB, 256 | +| Message Box | 500 messages, 4 MiB, 8 | 1,000 messages, 8 MiB, 24 | 5,000 messages, 32 MiB, 96 | +| Overlay Express | 500 lookup results, 4 MiB, 8 | 1,000 results, 8 MiB, 24 | 5,000 results, 32 MiB, 96 | +| UHRP Basic / Cloud | 500 records, 1 MiB, 16 | 1,000 records, 4 MiB, 64 | 5,000 records, 16 MiB, 250 | +| WAB | single-record APIs, 1 MiB, 64 | single-record APIs, 2 MiB, 128 | single-record APIs, 8 MiB, 256 | +| Wallet Storage API | 500 RPC rows, 4 MiB, 8 | 1,000 rows, 8 MiB, 24 | 5,000 rows, 32 MiB, 96 | + +Start with 512 MiB for `small`, 1 GiB for `standard`, and 8 GiB for +`high-throughput`. High-throughput is not a promise that every configured +maximum can run concurrently; measure the real record distribution and reduce +concurrency when response sizes approach their byte ceiling. + +## Service-specific controls + +| Service | Controls and defaults in `standard` | +| --- | --- | +| Chaintracks | `CHAINTRACKS_HEADERS_DEFAULT_LIMIT=1000`, `CHAINTRACKS_HEADERS_MAX_LIMIT=1000`; the static CDN streams files and has its own `CHAINTRACKS_CDN_*` connection policy. | +| Message Box | `MAX_MESSAGE_BODY_BYTES=1048576`, `MAX_RECIPIENTS=100`, `LIST_DEFAULT_LIMIT=1000`, `LIST_MAX_LIMIT=1000`, `LIST_MAX_OFFSET=100000`, `LIST_MAX_RESPONSE_BYTES=8388608`, inbox/sender quotas of 10,000 messages and 1 GiB, `MAX_ACKNOWLEDGMENT_IDS=1000`, device/permission page maximum 100, notification fan-out 100, and `RETENTION_DAYS=30`. `MESSAGE_LIST_BATCH_SIZE` remains a compatibility fallback. | +| Message Box maintenance/state | `AUTH_SESSION_TTL_MS=86400000`, `PAYMENT_REPLAY_TTL_DAYS=365`, `RETENTION_CLEANUP_INTERVAL_MS=900000`, `RETENTION_CLEANUP_BATCH_SIZE=1000`, `DB_POOL_MIN=0`, `DB_POOL_MAX=7`, and `DB_IDLE_TIMEOUT_MS=15000`. Auth sessions, quota locks, and payment replay records are shared in MySQL. | +| Overlay Express | `OVERLAY_MAX_LOOKUP_RESULTS=1000`, `MAX_BASM_TXIDS=1000`, `MAX_BASM_ANCHOR_RANGE=1000`, admin default/max pages 50/200, `JANITOR_BATCH_SIZE=250`, and `JANITOR_MAX_REPORT_RESULTS=1000`. Janitor scans every record through a cursor while retaining only the configured report detail. | +| UHRP Basic / Cloud | list default/max 200/1,000 and max offset 100,000; `MAX_FILE_BYTES=11000000000`, `MAX_RETENTION_MINUTES=525600`, JSON body 256 KiB, and upload body 64 MiB. Basic also bounds its MIME LRU at 10,000 entries. Downloads and uploads remain streamed. | +| WAB | 256 KiB JSON, 2 MiB response, MySQL pool min/max 2/10, and separate pre-auth, authentication, user, deletion, faucet, and share rate policies. Account-deletion state is database-backed. | +| Wallet Storage | RPC default/max rows 1,000/1,000, max request array items 1,000,000, 8 MiB RPC response, 8 MiB JSON, 8 MiB binary, and MySQL pool min/max 2/10 with configurable create/acquire/idle/reap/retry timeouts. Use `WALLET_INFRA_ROLE=api` for HTTP replicas and one `monitor` replica for background work. | + +All names above are appended to the service prefix where it is omitted in the +table. For example, Message Box `MAX_RECIPIENTS` means +`MESSAGE_BOX_MAX_RECIPIENTS`. + +Custom Overlay lookup services must also apply a database query limit. The +engine rejects a result formula above `OVERLAY_MAX_LOOKUP_RESULTS` before proof +hydration, but it cannot prevent custom service code from first materializing +an unsafe query internally. + +## Evidence and memory interpretation + +`governance/service-resource-profiles.json` is the machine-readable profile +contract. `pnpm resource-profiles:check` launches each of its 21 scenarios in a +fresh Node 24.18 process with a profile-constrained V8 heap. It constructs the +maximum representative page, retains an object graph, JSON text, and transport +bytes, and fails if that page exceeds the configured response cap or if the +three-copy concurrency model consumes more than 80% of profile memory. + +The 2026-08-04 Apple arm64 run produced these `standard` results: + +| Service | Representative maximum page | Measured RSS increase | Three-copy concurrency model | +| --- | ---: | ---: | ---: | +| Chaintracks | 157 KiB | 1.2 MiB | 29 MiB | +| Message Box | 2,001 KiB | 11.9 MiB | 141 MiB | +| Overlay Express | 4,001 KiB | 22.3 MiB | 281 MiB | +| UHRP Basic / Cloud | 1,001 KiB | 5.7 MiB | 188 MiB | +| WAB | 2 KiB | 0.1 MiB | 0.8 MiB | +| Wallet Storage | 4,001 KiB | 21.9 MiB | 281 MiB | + +The model uses fixed representative record sizes (160 B headers, 2 KiB +messages, 4 KiB overlay/wallet rows, and 1 KiB UHRP metadata). Real records, +authentication envelopes, database drivers, and telemetry differ. Before +raising a limit, replay production-shaped records under the intended cgroup +memory limit and preserve at least 20% RSS headroom. + +The incident-shaped validation also covered a 1,005-message inbox and confirms +that an omitted `limit` now returns at most 1,000 messages per server response. +Pagination metadata allows the Message Box client to retain historical +fetch-all behavior. Applications that do not want aggregate growth should set +`limit`, `pageSize`, and/or `maxPages`. + +Overlay Engine applies its lookup ceiling before hydration. Overlay Express +also pushes the same ceiling into its built-in SHIP and SLAP MongoDB queries, +using one overflow-probe row so a legacy `findAll` request fails clearly rather +than materializing an unbounded set or returning a silently incomplete answer. + +## Horizontal scaling + +Only Message Box, WAB, and Wallet Storage API have HPA examples in +`docs/examples/kubernetes/resource-profiles/`. CPU and memory targets are +starting signals, not capacity proof. + +Before scaling: + +- Message Box uses MySQL-backed auth sessions, payment replay protection, and + quota locks. Keep WebSockets disabled, use sticky sessions, or add an + operator-managed Socket.IO pub/sub adapter before serving a room from + multiple replicas. Enforce a shared gateway rate limit because the built-in + limiter is process-local. +- WAB account-deletion state and application state are shared in MySQL. Enforce + authentication/SMS abuse limits at a shared gateway so replica count does not + multiply the effective allowance. +- Run Wallet Storage HTTP pods with `WALLET_INFRA_ROLE=api`. Run exactly one + `WALLET_INFRA_ROLE=monitor` pod; never point an HPA at it. API sessions and + payment replay state use MySQL, while gateway rate limiting remains a + deployment responsibility. + +For all three, set CPU and memory requests, keep database pool maximum × maximum +replicas within database capacity, use readiness probes, and scale on request +saturation or queue/event-loop signals when the metrics platform supports +them. A memory target can add replicas for traffic growth but cannot repair a +leak; alert on monotonic per-pod RSS after traffic normalizes. + +## Message Box BRC-105 monetization + +Monetization is disabled by default. Set +`MESSAGE_BOX_MONETIZATION_ENABLED=true` to apply BRC-105 payment middleware on +authenticated routes. AuthFetch already implements the BRC-100 permissions and +BRC-105 exchange, so the Message Box client does not add a second cost-approval +layer. + +Legacy WebSocket sends remain available when monetization is disabled. On a +monetized server they receive `ERR_PAYMENT_REQUIRES_AUTHFETCH`, which makes the +Message Box Client immediately use its existing AuthFetch HTTP fallback. Live +sends are independently bounded by +`MESSAGE_BOX_WEBSOCKET_MAX_CONCURRENT_SENDS`, +`MESSAGE_BOX_WEBSOCKET_SEND_RATE_LIMIT` (per minute, per socket), and +`MESSAGE_BOX_WEBSOCKET_MAX_RECIPIENT_CONNECTIONS`. +HTTP sends also bound push work with +`MESSAGE_BOX_NOTIFICATION_RECIPIENT_CONCURRENCY` and +`MESSAGE_BOX_FCM_SEND_CONCURRENCY`; this prevents a valid multi-recipient send +from multiplying recipient and device fan-out into an unbounded promise set. + +| Variable | Default satoshis | Meaning | +| --- | ---: | --- | +| `MESSAGE_BOX_PRICE_BASE_SATOSHIS` | 50 | Fixed authenticated request component. | +| `MESSAGE_BOX_PRICE_PER_RECIPIENT_SATOSHIS` | 5 | Send fan-out component per recipient. | +| `MESSAGE_BOX_PRICE_PER_KIB_SATOSHIS` | 5 | UTF-8 message body component, rounded up by KiB. | +| `MESSAGE_BOX_PRICE_STORAGE_MIB_MONTH_SATOSHIS` | 1,000 | Retained payload component. | +| `MESSAGE_BOX_PRICE_LIST_PAGE_SATOSHIS` | 5 | Listing page component in addition to the base. | +| `MESSAGE_BOX_PRICE_UNLIMITED_RETENTION_MONTHS` | 12 | Up-front storage horizon when an operator explicitly configures unlimited retention. | +| `MESSAGE_BOX_ROUTE_PRICES_JSON` | `{}` | Absolute route-to-satoshi overrides; `0` makes a protected route free. | + +Recipient-configured delivery fees remain separate from the operator charge. +The operator price is paid and replay-checked before message fan-out; the send +transaction still preserves recipient remittance behavior. + +The defaults model a shared 1 vCPU / 2 GiB service and database allocation at +about USD 110/month, 10 million monthly protected requests, a 25% operating +reserve, equal send/list traffic, 1 KiB single-recipient sends retained for one +month, and a planning exchange rate of USD 25/BSV. That scenario recommends a +48-satoshi base; the rounded 50-satoshi default projects about USD 145/month. +It is a planning example, not a price feed or cloud quote. AWS bills Fargate by +allocated vCPU and memory and RDS by instance, storage, backup, I/O, and data +transfer; operators should replace every input with their bill and region: +[Fargate pricing](https://aws.amazon.com/fargate/pricing/), +[RDS for MySQL pricing](https://aws.amazon.com/rds/mysql/pricing/), and +[S3 pricing](https://aws.amazon.com/s3/pricing/). + +Run `node scripts/message-box-economics.mjs` and override its `MB_ECON_*` +variables to model traffic, margin, fixed costs, and a planning BSV/USD value. +The deployed pricing remains satoshi-native and never depends on that exchange +rate. Fee-model terminology follows the BSV documentation’s satoshi-per-KiB +convention: [BSV fee concepts](https://hub.bsvblockchain.org/bsv-skills-center/guides/sdks/concepts/fee). + +## Official-image downstream parity + +The official images now accept the generic runtime settings needed to replace +custom Babbage-derived images: + +| Workload | Upstream configuration now available | +| --- | --- | +| Message Box | Shared MySQL sessions/replay/quota locks, list/body/inbox/sender/retention limits, DB pool, Firebase and WebSocket controls, BRC-105 pricing, `/healthz`, and legacy `MESSAGE_LIST_BATCH_SIZE`. | +| WAB | DB pool, granular rate/resource policy, shared database deletion flow, `/healthz`, and leading-double-slash compatibility. | +| Wallet Storage | Raw or base64 JSON for `KNEX_DB_CONNECTION` and `FEE_MODEL`; raw or base64 admin keys; API/monitor role split; TAAL, WhatsOnChain, Bitails, Arcade, GorillaPool, and exchange-rate provider settings under `WALLET_STORAGE_*` with legacy aliases; logger level; DB/RPC/resource/payment controls. | + +Secrets, DNS, certificates, ingress, replica counts, provider credentials, and +cluster-specific shared rate limiting remain downstream. Migration should +compare the effective configuration in staging, then pin the official image by +digest and retain the previous custom digest for rollback. + +## Coordinated release order + +This pull request intentionally keeps the package and official-image changes in +one review unit, but release order still matters because standalone image lock +files cannot resolve package versions that have not yet been published: + +1. Merge the reviewed source revision and publish the public packages listed in + the coordinated release notes. +2. Refresh each standalone image lock from that same revision after the new + packages are available, and run its build, tests, resource-profile check, and + image smoke test. +3. Tag official images only after their resolved dependency tree contains the + coordinated package versions. Record the source commit and immutable image + digest together. +4. Exercise the standard profile in downstream staging, then canary Message + Box, WAB, and Wallet Storage in that order. Keep the previous image digest + and configuration available until the rollback window closes. + +Do not publish an image from this branch merely because its local standalone +tests pass: until the public packages exist, an unchanged lock can still select +the previous package implementation. Lock refresh and downstream rollout are +release/deployment steps after review, not additional functional PRs. diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 4e48da4f3..acbe6b6ce 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -48,26 +48,26 @@ authorized release action. | helpers | `create-bsv-app` | `1.0.4` | cli | cli | node | `>=22` | [packages/helpers/create-bsv-app](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/create-bsv-app) | | messaging | `@bsv/authsocket` | `2.1.5` | node-library | node-cjs, node-esm | node | `>=22` | [packages/messaging/authsocket](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/authsocket) | | messaging | `@bsv/authsocket-client` | `2.1.4` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/messaging/authsocket-client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/authsocket-client) | -| messaging | `@bsv/message-box-client` | `2.2.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/messaging/message-box-client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/message-box-client) | +| messaging | `@bsv/message-box-client` | `2.3.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/messaging/message-box-client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/message-box-client) | | messaging | `@bsv/paymail` | `2.4.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/messaging/ts-paymail](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/ts-paymail) | | middleware | `@bsv/402-pay` | `0.2.4` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/middleware/402-pay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/402-pay) | | middleware | `@bsv/auth` | `0.1.3` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/auth](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth) | -| middleware | `@bsv/auth-express-middleware` | `2.1.7` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/auth-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware) | +| middleware | `@bsv/auth-express-middleware` | `2.2.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/auth-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware) | | middleware | `@bsv/payment-express-middleware` | `2.1.5` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/payment-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/payment-express-middleware) | | network | `@bsv/teranode-listener` | `1.1.4` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.5` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | -| overlays | `@bsv/overlay` | `2.2.7` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | +| overlays | `@bsv/overlay` | `2.3.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | | overlays | `@bsv/overlay-discovery-services` | `2.1.6` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | -| overlays | `@bsv/overlay-express` | `2.4.9` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | +| overlays | `@bsv/overlay-express` | `2.5.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | | overlays | `@bsv/overlay-topics` | `1.6.8` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | | sdk | `@bsv/sdk` | `2.3.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.4` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.1.4` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.3` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | | wallet | `@bsv/wallet-relay` | `0.3.4` | cli-library | browser-bundler, browser-esm, cli, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/ts-wallet-relay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ts-wallet-relay) | -| wallet | `@bsv/wallet-toolbox` | `2.5.0` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | -| wallet | `@bsv/wallet-toolbox-client` | `2.5.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | -| wallet | `@bsv/wallet-toolbox-mobile` | `2.5.0` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | +| wallet | `@bsv/wallet-toolbox` | `2.6.0` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | +| wallet | `@bsv/wallet-toolbox-client` | `2.6.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | +| wallet | `@bsv/wallet-toolbox-mobile` | `2.6.0` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | ## Standalone infrastructure manifests @@ -76,7 +76,7 @@ the separately released and verified image digest. | Service | Package | Manifest version | Node engine | Runtime targets | Release | Source | | --- | --- | --- | --- | --- | --- | --- | -| BSV Chaintracks Server | `chaintracks-server` | `1.0.17` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/chaintracks-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/chaintracks-server) | +| BSV Chaintracks Server | `chaintracks-server` | `1.1.0` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/chaintracks-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/chaintracks-server) | | BSV Message Box Server | `@bsv/messagebox-server` | `1.1.24` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/message-box-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/message-box-server) | | BSV Overlay Server | `@bsv/overlay-express-examples` | `2.1.27` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/overlay-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/overlay-server) | | BSV UHRP Basic Server | `@bsv/uhrp-lite` | `0.1.20` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/uhrp-server-basic](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/uhrp-server-basic) | diff --git a/docs/superpowers/specs/2026-08-04-service-resource-safety-design.md b/docs/superpowers/specs/2026-08-04-service-resource-safety-design.md new file mode 100644 index 000000000..140036bc5 --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-service-resource-safety-design.md @@ -0,0 +1,212 @@ +--- +id: service-resource-safety-design +title: Service Resource Safety, Scaling, and Monetization — Design +kind: spec +domain: infra +version: 1.0.0 +last_updated: '2026-08-04' +last_verified: '2026-08-04' +status: experimental +tags: + - resource-safety + - containers + - autoscaling + - message-box + - payments +--- + +# Service Resource Safety, Scaling, and Monetization — Design + +**Date:** 2026-08-04 + +**Status:** Implemented on the draft service-resource-hardening change set + +**Scope:** The seven official TS Stack service images: Chaintracks, Message Box, Overlay, UHRP Basic, UHRP Cloud Bucket, WAB, and Wallet Infrastructure. + +## Context + +An input-size limit is not, by itself, an out-of-memory guarantee. A small request can select a large result, trigger expensive authenticated-response encoding, reserve durable storage, fan out to external systems, or start enough concurrent work to exhaust a process. Pagination can still be unsafe when a page is bounded by item count but not by encoded bytes. + +The official images need a consistent resource-safety contract with service-specific defaults. Operators must be able to tune that contract without rebuilding an image. The same contract must cover the client side when a convenience API can aggregate multiple bounded server responses into an unbounded in-memory result. + +This document records the architecture implemented by the remediation change set. Dependency-advisory work and downstream deployment are intentionally outside this change set. + +## Goals + +1. Make remotely initiated work, memory, and retained-state growth explicitly bounded in every official service image. +2. Provide conservative, service-specific defaults and documented environment/configuration overrides without requiring custom images. +3. Preserve existing deployments by making new controls additive, supporting established variable names, and providing explicit migration warnings before tightening behavior that callers may observe. +4. Give operators capacity-planning and horizontal-scaling guidance based on each service's actual state, leadership, CPU, memory, database, and connection behavior. +5. Expose Message Box operator monetization through the authenticated [BRC-105 HTTP service monetization flow](https://github.com/bsv-blockchain/BRCs/blob/master/payments/0105.md), using AuthFetch's existing BRC-100 permission path without a second client approval layer. +6. Make the official images capable of replacing known custom Message Box, WAB, and Wallet Storage images without embedding deployment-specific infrastructure in upstream. +7. Establish evidence strong enough to support a resource-safety claim: route inventory, constrained-heap tests, adversarial boundary tests, load/soak evidence, and release gates. + +## Non-goals + +- Shipping a single universal numeric limit across services with different work profiles. +- Automatically enabling horizontal scaling for a workload that still has process-local coordination or singleton responsibilities. +- Moving deployment-specific DNS, secrets, Kubernetes resources, or provider credentials into TS Stack. +- Dependency advisory, package-upgrade, or downstream rollout work. +- Changing payment protocol semantics from BRC-105 to BRC-121 or another 402 profile without an explicit compatibility decision. + +## Resource-safety contract + +Every remotely reachable operation and background loop must declare and enforce budgets in the following layers. + +| Layer | Required controls | Why request-byte limits are insufficient | +| ------------------ | ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| Transport | Header, JSON, binary, timeout, connection, and concurrent-request limits | Many small requests can exhaust a process even when each body is small. | +| Cardinality | Maximum items, ranges, recipients, identifiers, pages, and recursive work | A short query can select or generate a very large result. | +| Response | Encoded and authenticated response-byte ceiling, with streaming where the protocol permits | Serialization, signing, and transport can hold multiple copies of a response. | +| Retained state | Per-principal item and byte quotas, retention/expiry, and reservation accounting | An attacker can grow a victim's inbox or an operator's storage over time. | +| Dependencies | Database pool/queue limits, external-call concurrency, timeouts, retries, and circuit breaking | Fan-out transfers pressure to shared databases and providers. | +| Background work | Batch size, concurrency, schedule, backlog ceiling, and leader ownership | Maintenance and synchronization can compete with request traffic. | +| Client aggregation | Page, item, byte, host, and payment ceilings; lazy iteration for large collections | A client can recreate an unbounded result by accumulating safe pages. | +| Runtime | Container memory contract, V8 heap budget, graceful rejection, and overload telemetry | The process needs native-memory and serialization headroom beyond the JS heap. | + +Limits must be checked before expensive work. Where an exact response size cannot be known before querying, the implementation must use a database byte estimate, bounded chunks, a byte-counting encoder/stream, or a lower item ceiling backed by a tested maximum record size. A post-serialization check is defense in depth, not the only guard. + +### Configuration model + +Each image exposes: + +- a conservative service-specific default profile; +- optional named profiles such as `small`, `standard`, and `high-throughput`; +- granular environment overrides for every public limit; +- parser hard ceilings for accidental overflow, with explicit `-1`/`unlimited` operator opt-out for resource limits; +- startup validation that rejects internally inconsistent or unsafe settings; +- structured startup output containing effective non-secret limits and the selected profile; +- stable rejection codes and existing telemetry hooks for overload visibility. + +Existing variables remain valid. New variables use a consistent `__` pattern and are normalized through a shared typed configuration package. Examples include `MAX_JSON_BODY_BYTES`, `MAX_AUTHENTICATED_RESPONSE_BYTES`, `MAX_CONCURRENT_REQUESTS`, `REQUEST_TIMEOUT_MS`, `DB_POOL_MAX`, and route-specific item/byte/work limits under the established service prefix. + +Named profiles are convenience bundles, not substitutes for service-specific controls. Granular overrides win over the profile. The default profile must be safe inside the documented minimum container memory with tested headroom for native allocations, authentication, telemetry, and shutdown. + +Numeric defaults are recorded in `governance/service-resource-profiles.json` and exercised by `pnpm resource-profiles:check` so documentation and the capacity model can be reviewed together. + +## Service work and scaling model + +| Service | Dominant resource characteristics | Safe horizontal-scaling boundary | Required budget work | +| --------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Chaintracks | Header ingestion, range reads, encoding, disk/database I/O | Keep one ingest leader. Scale readers only with shared immutable/read-safe storage and explicit freshness behavior. | Bound ranges and encoded output; stream bulk formats; configure sync batches, retained files, pools, and reader concurrency. | +| Message Box | Authenticated encoding/signing, database rows and payload bytes, WebSockets, notification fan-out | Scale HTTP replicas only with shared auth sessions, shared rate/payment replay state, and a WebSocket routing or pub/sub strategy. | Bound pages by items and bytes; add inbox/sender quotas and retention; configure payload, fan-out, pools, notification concurrency, and signed-response size. | +| Overlay | Lookup-dependent result expansion, transaction proof material, GASP/BASM synchronization, maintenance | Keep stateful background roles single-owner until leader election or role separation exists. Query APIs can scale against safe shared state. | Bound lookup/proof/range work and responses; configure enabled services, background batches/concurrency, pools, and verbose diagnostics. | +| UHRP Basic | Streaming upload/download, filesystem capacity, metadata listing, MIME inspection | A single writer is the default. Multiple replicas require a concurrency-safe shared filesystem and shared quotas/rate state. | Bound list/search results, retention, account/object bytes, upload reservations, file inspection, and filesystem concurrency. | +| UHRP Cloud Bucket | Presigned object transfer, metadata/database work, cloud-provider calls | API replicas can scale when rate/quota/replay state is shared and the object provider supports concurrent operation. | Bound list/search results, retention, reservations, provider concurrency/retries, and durable account/object quotas. | +| WAB | Small auth requests, database/session work, abuse-sensitive account state | Scale when rate limits, sessions, and account-state guards are shared and database capacity is measured. | Configure shared rate stores, pools, request concurrency, account entity quotas, and stable readiness behavior. | +| Wallet Infrastructure | Authenticated RPC, transaction/proof expansion, database work, monitor/background jobs | Split a singleton monitor/worker role from scalable API replicas, with shared sessions, replay/rate state, and storage. | Validate every RPC at the transport boundary; add method-specific work/response budgets, pool controls, provider concurrency, and monitor leadership. | + +HPA examples are provided only for Message Box, WAB, and the Wallet Storage API role. CPU alone is not a sufficient scaling signal. Guidance includes memory working set, request saturation, database pool capacity, WebSocket state, and background-role ownership as applicable. Every example has a bounded replica range, stabilization policy, and stated shared-state prerequisite; deployment-owned manifests retain termination and disruption settings. + +## Message Box server and client + +### Server invariants + +Message listing is bounded simultaneously by item count and encoded bytes. Stored message payloads have a configurable per-message ceiling, while each sender and recipient has configurable outstanding-item, outstanding-byte, and retention ceilings. Writes reserve quota atomically before committing so concurrent requests cannot oversubscribe the account. + +Authenticated response middleware enforces a response budget early enough to avoid repeated full-size copies during JSON encoding and signing. The same reusable defense applies to other BRC-103 HTTP services. Database pool sizing, query timeout, notification fan-out, WebSocket connection limits, session storage, rate limiting, and payment replay storage are operator-configurable. + +The standalone image exposes the shared session and abuse-control adapters already supported by the composable packages. A database-backed baseline avoids requiring a second datastore; an optional shared low-latency adapter may be provided for larger installations. + +### Client invariants + +The Message Box client preserves the historical fetch-all convenience API by following server pagination. Callers can set `offset`/`skip`, a total `limit`, `pageSize`, or `maxPages`; omission preserves fetch-all compatibility. Decryption, payment internalization, and host fan-out retain bounded concurrency while every server response is independently capped. + +## BRC-105 monetization + +The standalone Message Box image currently has the components needed for authenticated payment middleware, but operator pricing must be a supported runtime configuration rather than application code. Monetization is off by default for backward compatibility. + +The implemented configuration surface supports: + +- BRC-105 enablement and an AuthFetch-compatible 402 challenge; +- operator base price per protected request; +- per-recipient and per-KiB delivery components; +- optional storage/retention tiers or prepaid quota; +- route-specific free tiers and prices; +- quote lifetime, price floor/ceiling, and payment replay persistence; +- AuthFetch's existing BRC-105 payment exchange without a duplicate Message Box approval mechanism; +- separate operator pricing and recipient-configured delivery fees. + +Operator request pricing and recipient delivery fees are distinct ledgers. The implementation must define one canonical calculation and presentation path so enabling BRC-105 cannot accidentally double-charge the send operation. Payments are validated before costly message fan-out and quota is reserved before payment is finalized. Failure and refund semantics must be documented for partial downstream failure. + +### Economic model + +The reference calculator uses operator-supplied costs and a replaceable planning exchange rate rather than a runtime price feed: + +```text +monthly_required_revenue = + fixed_compute + + fixed_database + + high_availability_overhead + + observability_and_backups + + variable_storage + + variable_egress + + variable_provider_calls + + wallet_and_payment_processing + + operating_reserve + +marginal_message_cost = + authentication_and_signing + + database_writes_and_expected_reads + + payload_bytes * retention_duration * replicated_storage_rate + + expected_egress + + notification_fanout + + payment_internalization + + observability + +price_satoshis = ceil((allocated_cost_fiat * 100_000_000) / reference_bsv_price_fiat) + + margin_satoshis +``` + +`scripts/message-box-economics.mjs` models payload size, recipient count, retention, send/list mix, request volume, fixed cost, and operating margin. Deployed prices remain satoshi-native with no exchange-rate dependency. + +## Official-image parity for downstream migration + +Runtime behavior that is generally useful belongs upstream; environment-specific deployment wiring remains downstream. + +| Workload | Generic upstream capabilities needed before migration | Remains downstream | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| Message Box | Shared database-backed BRC-103 sessions, configurable list/byte/retention quotas, health/readiness contract, WebSocket and notification settings, payment/replay settings, pool and concurrency controls | DNS, certificates, Kubernetes manifests, secrets, provider-specific notification credentials | +| WAB | Shared rate/session stores, deletion/account-state protections, request correlation, stable health/readiness, database pool controls, narrowly scoped compatibility for currently released clients | Cluster topology, disruption policy, ingress, secrets | +| Wallet Storage | Shared BRC-103 sessions, monitor/API role separation, provider URL/key settings, admin identities, logging controls, health/readiness, RPC budgets, database/provider pools | Provider secret values, cluster workloads, monitor scheduling, DNS, certificates | + +Migration validation must compare effective configuration and behavior, not merely environment variable names. Staging should exercise the official image with production-shaped limits before a canary or production cutover. Rollback remains the previous digest and configuration until the retention window closes. + +## Verification and confidence gates + +A high-confidence resource-safety claim requires all of the following: + +1. A machine-readable inventory of every public route, authenticated RPC method, WebSocket path, scheduled loop, and worker, with its budgets and state ownership. +2. Schema validation and boundary tests for every item/range/byte/duration/concurrency input, including direct RPC calls that bypass SDK convenience validators. +3. Constrained-heap tests that exercise maximum legal records, response encoding, authenticated signing, fan-out, and concurrent boundary traffic. Tests must assert bounded resident memory and graceful rejection above the budget. +4. Fuzz/property tests for numeric overflow, negative/NaN values, array cardinality, nested JSON, compression expansion, and retention arithmetic. +5. Load and soak tests for each documented resource profile, recording p95/p99 latency, peak RSS, heap, GC, event-loop lag, database saturation, response bytes, backlog, and rejection rate. +6. Client tests across multiple hosts proving that page accumulation, decryption, and payment processing honor aggregate budgets. +7. Deployment tests with container memory limits and a V8 heap budget that leaves measured native headroom. +8. CI checks that fail when a route or RPC is added without a resource contract, or when generated operations documentation drifts. +9. Maintainer review of the complete remediation and coordinated release notes before merge. + +No service should be described as OOM-proof. The supported claim is that all known remotely controllable resource dimensions are bounded, tested under the documented envelope, observable, and rejected safely when exhausted. + +## Delivery sequence + +1. Land reusable response-budget/authentication defenses. +2. Add the machine-readable operation/resource schema, shared parsing, profiles, startup validation, and generated docs. +3. Harden Message Box server and client, including retained-state quotas and bounded client iteration. +4. Expose BRC-105 pricing/replay configuration and add server/client payment tests. +5. Apply route/RPC/background-work budgets to Chaintracks, Overlay, UHRP, WAB, and Wallet Infrastructure on the same coordinated branch. +6. Add role separation and shared-state adapters needed for replica safety, then publish service-specific HPA guidance and examples. +7. Produce the economics worksheet and profile benchmark reports. +8. Validate official images in downstream staging, migrate Message Box, then WAB, then Wallet Storage, and record evidence before retiring custom images. + +The coordinated implementation remains one draft pull request. Image release and downstream migration remain separate, deliberate actions after review and merge. + +## Resolved decisions + +- Server pages default to and cap at 1,000 Message Box messages; the client preserves fetch-all pagination. +- Operators may explicitly set resource limits to `-1`/`unlimited`; omission always uses a bounded profile default. +- MySQL is the shared-state baseline. Optional stores remain injectable at package boundaries where already supported; a mandatory Redis dependency is not introduced. +- The default profile targets at least 1 GiB, with small and high-throughput profiles backed by the checked-in model. +- Message Box pricing is satoshi-native, disabled by default, and uses AuthFetch without a second approval layer. +- `/healthz` and leading-double-slash compatibility are permanent additive behavior. +- Wallet Infrastructure uses scalable `api` roles and one `monitor` role. diff --git a/governance/browser-artifact-policy.json b/governance/browser-artifact-policy.json index 9ee419686..c22e83810 100644 --- a/governance/browser-artifact-policy.json +++ b/governance/browser-artifact-policy.json @@ -101,7 +101,7 @@ "path": "packages/wallet/wallet-toolbox/client", "budget": "packages/wallet/wallet-toolbox/client/platform-budget.json", "entry": ".", - "splittingDisposition": "The dedicated browser export excludes Node storage and database adapters and is verified through its platform-specific exact-package gate." + "splittingDisposition": "The dedicated browser export excludes Node storage, database adapters, and direct P2P. Its browser-safe fetch/SSE ChainTracks adapter is an intentional shared capability measured by the platform-specific exact-package gate." } ] } diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index e1478d437..9b89d1c00 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "lastReviewed": "2026-08-03", + "lastReviewed": "2026-08-04", "owner": "ts-stack-maintainers", "entries": [ { @@ -34,9 +34,9 @@ { "name": "@bsv/auth-express-middleware", "publishedVersion": "2.1.2", - "releaseType": "patch", - "summary": "Standardizes package quality and strengthens authenticated Express session, edge-policy, and error-handling boundaries, including containment of both synchronous and asynchronous BRC-103 callback failures, and restores one shared Express 4/5 runtime and type graph for consumers.", - "migration": "No consumer migration is required; existing public CORS defaults and middleware APIs are retained, authentication callback failures now produce a controlled HTTP error, and Express 4 and 5 applications use their own peer-provided Express installation." + "releaseType": "minor", + "summary": "Bounds BRC-104 application-response capture before authenticated signing, with an 8 MiB default, a configurable transport limit, stable 413 errors, and boundary regression coverage.", + "migration": "Existing middleware construction remains compatible. Operators may set transportLimits.maxResponseBytes to a positive byte count or -1 when the embedding service enforces an equivalent response budget." }, { "name": "@bsv/authsocket", @@ -97,16 +97,16 @@ { "name": "@bsv/message-box-client", "publishedVersion": "2.2.2", - "releaseType": "patch", - "summary": "Adds strict package contracts and hardens PeerPay parsing, proof handling, cancellation, classification, and acknowledgement flows.", - "migration": "No consumer migration is required; Message Box protocol and client entry points are unchanged." + "releaseType": "minor", + "summary": "Follows bounded Message Box server pagination while preserving historical fetch-all behavior and adds offset/skip, total-limit, page-size, and optional page-ceiling controls.", + "migration": "Existing listMessages and listMessagesLite calls continue to fetch all available messages. Applications that need an aggregate memory ceiling should set limit and/or maxPages; no additional BRC-105 approval callback is required because AuthFetch uses wallet permissions." }, { "name": "@bsv/overlay", "publishedVersion": "2.2.1", - "releaseType": "patch", - "summary": "Repairs the documented storage export and decomposes Engine submission validation, broadcast, storage, notification, and propagation orchestration while preserving execution order and behavior.", - "migration": "No consumer migration is required; existing imports, submission results, notification contracts, storage order, and network behavior remain unchanged." + "releaseType": "minor", + "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration so a compact remote query cannot amplify into unbounded retained work.", + "migration": "Existing Engine constructor calls remain valid and default to 1,000 lookup formulas. Pass -1 as the final maxLookupResults argument only when a custom lookup service and deployment enforce an equivalent bound." }, { "name": "@bsv/overlay-discovery-services", @@ -118,9 +118,9 @@ { "name": "@bsv/overlay-express", "publishedVersion": "2.4.2", - "releaseType": "patch", - "summary": "Adds strict package and edge-policy contracts, restores declaration-safe exports, adds idempotent shutdown, and hardens provider-chain failure handling and synchronization configuration.", - "migration": "No consumer migration is required; wildcard credential-free public access remains the default and runtimes may opt into the new close method." + "releaseType": "minor", + "summary": "Adds small, standard, and high-throughput resource profiles; bounded lookup, BASM, admin, connection, body, and response work; and streaming janitor scans with independently capped report retention.", + "migration": "The standard profile is selected by default. Existing configuration methods remain compatible; use OVERLAY_RESOURCE_PROFILE or granular OVERLAY_* limits, and ensure custom lookup services bound their own database queries before returning formulas." }, { "name": "@bsv/overlay-topics", @@ -194,24 +194,24 @@ }, { "name": "@bsv/wallet-toolbox", - "publishedVersion": "2.4.22", + "publishedVersion": "2.5.0", "releaseType": "minor", - "summary": "Makes the successful fragmented createAction path atomic and set-based, overlaps batched proof reads with persistence, batch-validates compound proofs and canonical P2PKH signatures, shares BRC-42 derivation work, removes unused commit reads, bulk-inserts outputs, coalesces authenticated timestamp-only Knex session touches, adds timings for every remaining material phase, and makes UMP account lookup resilient to partial overlay failure.", - "migration": "No API or persistence migration is required. Storage-provider additions are backward-compatible with fallbacks and existing databases use the normal migration path. Applications may now enter new-user flow when at least one overlay host returns a clean empty result despite malformed or unavailable peers; one verified token still establishes an existing account, multiple unresolved verified tokens remain an error, and WAB existing-account continuity still blocks replacement-wallet onboarding. Wallet results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are otherwise unchanged." + "summary": "Adds resource-profile-aware Wallet Storage RPC pagination, array and response budgets, bounded authenticated response capture, durable payment replay and session schemas, and double-slash compatibility; makes fragmented createAction atomic and set-based with batched proof reads, validation, output writes, shared derivation work, coalesced session touches, and complete privacy-safe phase timings; makes UMP account lookup resilient to partial overlay failure; and refreshes ChainTracks with credential-free Arcade/go-chaintracks bulk and SSE sources, five-network genesis correctness, prioritized failover, local last-good operation, source health, browser-safe reconnects, and a rate-limited keyless WhatsOnChain fallback.", + "migration": "List/find RPC calls that omit a limit now receive the profile default (1,000 in standard), and larger caller limits are rejected above the operator maximum. Existing calls and storage data remain compatible; migrations add auth_sessions, payment_replays, and createAction indexes automatically, and storage-provider additions retain backward-compatible fallbacks. Applications may now enter new-user flow when at least one overlay host returns a clean empty result despite malformed or unavailable peers; one verified token still establishes an existing account, multiple unresolved verified tokens remain an error, and WAB existing-account continuity still blocks replacement-wallet onboarding. The default ChainTracks client for mainnet, testnet, and TerraTestNet now uses each public Arcade v2 endpoint without a credential; explicit ChaintracksClientApi injection and legacy v1 URLs remain supported. STN and Terra Scaling TestNet require an explicit STN_CHAINTRACKS_URL/TSTN_CHAINTRACKS_URL or corresponding Arcade URL. A WhatsOnChain key is optional, anonymous fallback stays below the documented public rate, and ChainTracks header/info requests retry anonymously when a configured key is rejected. Browser/mobile CORS expectations, wallet results, BRC-103/104, AuthFetch, JSON-RPC, and wallet wire behavior are otherwise unchanged." }, { "name": "@bsv/wallet-toolbox-client", - "publishedVersion": "2.4.22", + "publishedVersion": "2.5.0", "releaseType": "minor", - "summary": "Carries the lockstep browser build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, and resilient UMP account lookup.", - "migration": "No API or persistence migration is required. Browser wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox; older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged." + "summary": "Carries the lockstep browser build with the bounded remote-storage contract, batched proof assembly, linear funding and signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, resilient UMP account lookup, and the browser-safe credential-free ChainTracks v2 client with reconnecting SSE and five-network header support.", + "migration": "No browser consumer migration is required. Remote storage callers may provide explicit list limits when they need a value other than the operator profile default, and browser wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox. Mainnet, testnet, and TerraTestNet ChainTracks defaults now use public credential-free Arcade v2 endpoints; explicit clients and legacy v1 URLs remain compatible, while STN and Terra Scaling TestNet require an explicit endpoint. Older compatible SDK peers retain the validated sequential proof fallback, IndexedDB upgrades automatically, and CORS, BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage wire behavior remain compatible." }, { "name": "@bsv/wallet-toolbox-mobile", - "publishedVersion": "2.4.22", + "publishedVersion": "2.5.0", "releaseType": "minor", - "summary": "Carries the lockstep mobile build with batched proof assembly, linear funding/signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, and resilient UMP account lookup.", - "migration": "No API or persistence migration is required. Mobile wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox; older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged." + "summary": "Carries the lockstep mobile build with the bounded remote-storage contract, batched proof assembly, linear funding and signing work, canonical P2PKH verification, expired-reservation filtering, complete privacy-safe createAction timings, resilient UMP account lookup, and the mobile-safe credential-free ChainTracks v2 client with reconnecting SSE and five-network header support.", + "migration": "No mobile consumer migration is required. Remote storage callers may provide explicit list limits when they need a value other than the operator profile default, and mobile wallets receive the resilient partial-host UMP lookup behavior described for @bsv/wallet-toolbox. Mainnet, testnet, and TerraTestNet ChainTracks defaults now use public credential-free Arcade v2 endpoints; explicit clients and legacy v1 URLs remain compatible, while STN and Terra Scaling TestNet require an explicit endpoint. Older compatible SDK peers retain the validated sequential proof fallback, and BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage wire behavior remain compatible." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index ab9e026bb..5e439ffd4 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -308,25 +308,25 @@ "@bsv/templates": "1.9.6", "@bsv/authsocket": "2.1.5", "@bsv/authsocket-client": "2.1.4", - "@bsv/message-box-client": "2.2.6", + "@bsv/message-box-client": "2.3.0", "@bsv/paymail": "2.4.6", "@bsv/402-pay": "0.2.4", "@bsv/auth": "0.1.3", - "@bsv/auth-express-middleware": "2.1.7", + "@bsv/auth-express-middleware": "2.2.0", "@bsv/payment-express-middleware": "2.1.5", "@bsv/teranode-listener": "1.1.4", "@bsv/gasp": "1.3.5", - "@bsv/overlay": "2.2.7", + "@bsv/overlay": "2.3.0", "@bsv/overlay-discovery-services": "2.1.6", - "@bsv/overlay-express": "2.4.9", + "@bsv/overlay-express": "2.5.0", "@bsv/overlay-topics": "1.6.8", "@bsv/sdk": "2.3.0", "@bsv/verifast": "0.3.4", "@bsv/btms": "1.1.4", "@bsv/btms-permission-module": "1.1.3", "@bsv/wallet-relay": "0.3.4", - "@bsv/wallet-toolbox-client": "2.5.0", - "@bsv/wallet-toolbox-mobile": "2.5.0", - "@bsv/wallet-toolbox": "2.5.0" + "@bsv/wallet-toolbox-client": "2.6.0", + "@bsv/wallet-toolbox-mobile": "2.6.0", + "@bsv/wallet-toolbox": "2.6.0" } } diff --git a/governance/repository-health/projects.json b/governance/repository-health/projects.json index 370b08ebf..c5b6e5c93 100644 --- a/governance/repository-health/projects.json +++ b/governance/repository-health/projects.json @@ -155,6 +155,24 @@ "generator": "scripts/sync-service-edge-policy.mjs", "analysisPolicy": "exclude-generated", "reviewPolicy": "Review and analyze the canonical WAB policy, verify byte-for-byte synchronization in CI, and exclude only intentional generated-copy duplication." + }, + { + "path": "infra/uhrp-server-cloud-bucket/src/resourceLimits.ts", + "owner": "ts-stack-maintainers", + "sourceInputs": ["infra/uhrp-server-basic/src/resourceLimits.ts"], + "generator": "scripts/sync-service-runtime-copies.mjs", + "analysisPolicy": "exclude-generated", + "reviewPolicy": "Review and analyze the canonical UHRP resource limits, verify byte-for-byte synchronization in CI, and exclude only intentional generated-copy duplication." + }, + { + "path": "infra/wallet-infra/src/KnexPaymentReplayStore.ts", + "owner": "ts-stack-maintainers", + "sourceInputs": [ + "packages/wallet/wallet-toolbox/src/storage/remoting/KnexPaymentReplayStore.ts" + ], + "generator": "scripts/sync-service-runtime-copies.mjs", + "analysisPolicy": "exclude-generated", + "reviewPolicy": "Review and analyze the canonical wallet payment replay store, verify byte-for-byte synchronization in CI, and exclude only intentional generated-copy duplication." } ], "profiles": { diff --git a/governance/service-operations.json b/governance/service-operations.json index 2ce082774..f05322130 100644 --- a/governance/service-operations.json +++ b/governance/service-operations.json @@ -1,6 +1,6 @@ { "schemaVersion": 2, - "lastReviewed": "2026-07-29", + "lastReviewed": "2026-08-05", "owner": "ts-stack-maintainers", "policy": { "publicEdge": { @@ -85,16 +85,25 @@ "path": "infra/chaintracks-server", "envExample": "infra/chaintracks-server/.env.example", "port": "PORT (default 3011; CDN is port + 1)", - "livenessPath": "/getInfo", - "readinessPath": "/getInfo", + "livenessPath": "/healthz", + "readinessPath": "/readyz", "configuration": { "required": ["CHAIN"], "optional": [ "BULK_HEADERS_PATH", "CDN_HOST_URL", + "CHAINTRACKS_DISABLE_WHATSONCHAIN", + "CHAINTRACKS_UPSTREAM_API_PREFIX", + "CHAINTRACKS_UPSTREAM_MAX_HEADERS", + "CHAINTRACKS_UPSTREAM_URL", "ENABLE_BULK_HEADERS_CDN", "PORT", + "ROUTING_PREFIX", "SOURCE_CDN_URL", + "STN_ARCADE_URL", + "STN_CHAINTRACKS_URL", + "TSTN_ARCADE_URL", + "TSTN_CHAINTRACKS_URL", "WHATSONCHAIN_API_KEY" ], "secrets": ["OTEL_EXPORTER_OTLP_HEADERS", "WHATSONCHAIN_API_KEY"] @@ -104,7 +113,14 @@ "telemetryFile": "src/telemetry.ts", "loggerFile": "src/logger.ts", "preload": "--require ./dist/telemetry.js", - "operations": ["listen", "shutdown", "headers.export", "headers.ingest"] + "operations": [ + "listen", + "shutdown", + "headers.export", + "headers.ingest", + "readiness", + "config.summary" + ] }, "criticalJourneys": [ "return current chain information", @@ -113,7 +129,7 @@ ], "alerts": [ "header tip age or height stops advancing", - "bulk-header export or upstream retrieval repeatedly fails", + "all configured bulk/live sources are degraded or upstream retrieval repeatedly fails", "API or CDN saturation exceeds its independent concurrency budget" ], "state": "Bulk-header files under BULK_HEADERS_PATH; upstream headers are reproducible.", @@ -138,7 +154,7 @@ "path": "infra/message-box-server", "envExample": "infra/message-box-server/.env.example", "port": "PORT, then HTTP_PORT (default 8080)", - "livenessPath": "/health", + "livenessPath": "/healthz", "readinessPath": "/ready", "configuration": { "required": ["SERVER_PRIVATE_KEY", "WALLET_STORAGE_URL"], @@ -199,7 +215,7 @@ "path": "infra/overlay-server", "envExample": "infra/overlay-server/.env.example", "port": "8080", - "livenessPath": "/health/live", + "livenessPath": "/healthz", "readinessPath": "/health/ready", "configuration": { "required": [ @@ -277,7 +293,7 @@ "path": "infra/uhrp-server-basic", "envExample": "infra/uhrp-server-basic/.env.example", "port": "HTTP_PORT (default 8080)", - "livenessPath": "/health", + "livenessPath": "/healthz", "readinessPath": "/ready", "configuration": { "required": ["BSV_NETWORK", "SERVER_PRIVATE_KEY", "WALLET_STORAGE_URL"], @@ -323,7 +339,7 @@ "path": "infra/uhrp-server-cloud-bucket", "envExample": "infra/uhrp-server-cloud-bucket/secrets/.env.example", "port": "HTTP_PORT (default 8080)", - "livenessPath": "/health", + "livenessPath": "/healthz", "readinessPath": "/ready", "configuration": { "required": [ @@ -381,7 +397,7 @@ "path": "infra/wab", "envExample": "infra/wab/.env.example", "port": "PORT (default 8080)", - "livenessPath": "/info", + "livenessPath": "/healthz", "readinessPath": "/info", "configuration": { "required": [ @@ -450,7 +466,7 @@ "path": "infra/wallet-infra", "envExample": "infra/wallet-infra/.env.example", "port": "HTTP_PORT (default 8081; samples set 8080 without nginx)", - "livenessPath": "/", + "livenessPath": "/healthz", "readinessPath": "/", "configuration": { "required": ["BSV_NETWORK", "KNEX_DB_CONNECTION", "SERVER_PRIVATE_KEY"], diff --git a/governance/service-resource-profiles.json b/governance/service-resource-profiles.json new file mode 100644 index 000000000..8ad3b74cb --- /dev/null +++ b/governance/service-resource-profiles.json @@ -0,0 +1,184 @@ +{ + "schemaVersion": 1, + "lastVerified": "2026-08-04", + "measurement": { + "node": "24.18.0", + "duplicationFactor": 3, + "description": "Representative page retained as object graph, JSON text, and authenticated transport bytes. The benchmark is a capacity model, not a substitute for a database-backed soak test." + }, + "profiles": { + "small": { "minimumMemoryMiB": 512, "recommendedVcpu": 0.5 }, + "standard": { "minimumMemoryMiB": 1024, "recommendedVcpu": 1 }, + "high-throughput": { "minimumMemoryMiB": 8192, "recommendedVcpu": 4 } + }, + "services": { + "chaintracks": { + "environmentPrefix": "CHAINTRACKS", + "representativeItemBytes": 160, + "values": { + "small": { + "defaultItems": 250, + "maxItems": 500, + "maxResponseBytes": 1048576, + "concurrency": 32 + }, + "standard": { + "defaultItems": 1000, + "maxItems": 1000, + "maxResponseBytes": 4194304, + "concurrency": 64 + }, + "high-throughput": { + "defaultItems": 5000, + "maxItems": 5000, + "maxResponseBytes": 33554432, + "concurrency": 256 + } + } + }, + "message-box": { + "environmentPrefix": "MESSAGE_BOX", + "representativeItemBytes": 2048, + "values": { + "small": { + "defaultItems": 250, + "maxItems": 500, + "maxResponseBytes": 4194304, + "concurrency": 8 + }, + "standard": { + "defaultItems": 1000, + "maxItems": 1000, + "maxResponseBytes": 8388608, + "concurrency": 24 + }, + "high-throughput": { + "defaultItems": 1000, + "maxItems": 5000, + "maxResponseBytes": 33554432, + "concurrency": 96 + } + } + }, + "overlay": { + "environmentPrefix": "OVERLAY", + "representativeItemBytes": 4096, + "values": { + "small": { + "defaultItems": 500, + "maxItems": 500, + "maxResponseBytes": 4194304, + "concurrency": 8 + }, + "standard": { + "defaultItems": 1000, + "maxItems": 1000, + "maxResponseBytes": 8388608, + "concurrency": 24 + }, + "high-throughput": { + "defaultItems": 5000, + "maxItems": 5000, + "maxResponseBytes": 33554432, + "concurrency": 96 + } + } + }, + "uhrp-basic": { + "environmentPrefix": "UHRP", + "representativeItemBytes": 1024, + "values": { + "small": { + "defaultItems": 100, + "maxItems": 500, + "maxResponseBytes": 1048576, + "concurrency": 16 + }, + "standard": { + "defaultItems": 200, + "maxItems": 1000, + "maxResponseBytes": 4194304, + "concurrency": 64 + }, + "high-throughput": { + "defaultItems": 1000, + "maxItems": 5000, + "maxResponseBytes": 16777216, + "concurrency": 250 + } + } + }, + "uhrp-cloud-bucket": { + "environmentPrefix": "UHRP", + "representativeItemBytes": 1024, + "values": { + "small": { + "defaultItems": 100, + "maxItems": 500, + "maxResponseBytes": 1048576, + "concurrency": 16 + }, + "standard": { + "defaultItems": 200, + "maxItems": 1000, + "maxResponseBytes": 4194304, + "concurrency": 64 + }, + "high-throughput": { + "defaultItems": 1000, + "maxItems": 5000, + "maxResponseBytes": 16777216, + "concurrency": 250 + } + } + }, + "wab": { + "environmentPrefix": "WAB", + "representativeItemBytes": 2048, + "values": { + "small": { + "defaultItems": 1, + "maxItems": 1, + "maxResponseBytes": 1048576, + "concurrency": 64 + }, + "standard": { + "defaultItems": 1, + "maxItems": 1, + "maxResponseBytes": 2097152, + "concurrency": 128 + }, + "high-throughput": { + "defaultItems": 1, + "maxItems": 1, + "maxResponseBytes": 8388608, + "concurrency": 256 + } + } + }, + "wallet-storage": { + "environmentPrefix": "WALLET_STORAGE", + "representativeItemBytes": 4096, + "values": { + "small": { + "defaultItems": 500, + "maxItems": 500, + "maxResponseBytes": 4194304, + "concurrency": 8 + }, + "standard": { + "defaultItems": 1000, + "maxItems": 1000, + "maxResponseBytes": 8388608, + "concurrency": 24 + }, + "high-throughput": { + "defaultItems": 1000, + "maxItems": 5000, + "maxResponseBytes": 33554432, + "concurrency": 96 + } + } + } + } +} diff --git a/governance/service-runtime-copy-policy.json b/governance/service-runtime-copy-policy.json new file mode 100644 index 000000000..134c52e7f --- /dev/null +++ b/governance/service-runtime-copy-policy.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "lastReviewed": "2026-08-04", + "owner": "ts-stack-maintainers", + "rationale": "Standalone image build contexts retain a small number of runtime sources that are canonically owned elsewhere. These copies are synchronized byte-for-byte so published packages and official images cannot drift.", + "copies": [ + { + "canonicalSource": "infra/uhrp-server-basic/src/resourceLimits.ts", + "synchronizedSources": ["infra/uhrp-server-cloud-bucket/src/resourceLimits.ts"] + }, + { + "canonicalSource": "packages/wallet/wallet-toolbox/src/storage/remoting/KnexPaymentReplayStore.ts", + "synchronizedSources": ["infra/wallet-infra/src/KnexPaymentReplayStore.ts"] + } + ] +} diff --git a/infra/chaintracks-server/.env.docker b/infra/chaintracks-server/.env.docker index 11ff6cd6c..5e70bac2e 100644 --- a/infra/chaintracks-server/.env.docker +++ b/infra/chaintracks-server/.env.docker @@ -1,13 +1,23 @@ # Docker Environment Configuration # Copy this file to .env and customize for your deployment -# Chain: 'main' or 'test' +# Chain: main | test | stn | ttn | tstn CHAIN=main -# WhatsOnChain API Key (optional but recommended for production) -# Get your API key at: https://whatsonchain.com/ +# Optional higher-limit key. ChainTracks works keylessly by default. WHATSONCHAIN_API_KEY= +# Credential-free public defaults exist for main/test/ttn. Required for stn or +# tstn unless the matching *_ARCADE_URL is configured. +CHAINTRACKS_UPSTREAM_URL= +CHAINTRACKS_UPSTREAM_API_PREFIX= +CHAINTRACKS_UPSTREAM_MAX_HEADERS=1000 +CHAINTRACKS_DISABLE_WHATSONCHAIN=false +STN_ARCADE_URL= +STN_CHAINTRACKS_URL= +TSTN_ARCADE_URL= +TSTN_CHAINTRACKS_URL= + # SOURCE_CDN_URL - Remote CDN to download bulk headers FROM (fallback when local files don't exist) SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ diff --git a/infra/chaintracks-server/.env.example b/infra/chaintracks-server/.env.example index 3c0329a0b..162af3b56 100644 --- a/infra/chaintracks-server/.env.example +++ b/infra/chaintracks-server/.env.example @@ -1,6 +1,6 @@ # ChaintracksService Configuration -# Chain: 'main' or 'test' +# Chain: main | test | stn | ttn | tstn CHAIN=main # Server port (default: 3011) @@ -10,10 +10,25 @@ PORT=3011 # Example: /api/v1 results in /api/v1/getInfo ROUTING_PREFIX= -# WhatsOnChain API Key (optional but recommended for production) -# Get your API key at: https://whatsonchain.com/ +# Optional WhatsOnChain key for higher mainnet limits. ChainTracks works +# anonymously below the documented 3 requests/second public limit. WoC is not +# used for stn, ttn, or tstn. WHATSONCHAIN_API_KEY= +# Preferred Arcade/go-chaintracks v2 source. Mainnet, testnet, and TTN use +# public defaults when this is empty. Set an explicit URL for STN/TSTN, or set +# STN_ARCADE_URL/TSTN_ARCADE_URL. Use "disabled" or "none" to disable it. +CHAINTRACKS_UPSTREAM_URL= +# Inferred as /chaintracks/v2 unless the URL already ends in /v2. +CHAINTRACKS_UPSTREAM_API_PREFIX= +CHAINTRACKS_UPSTREAM_MAX_HEADERS=1000 +# Mainnet/testnet only. The default keyless fallback is rate limited. +CHAINTRACKS_DISABLE_WHATSONCHAIN=false +STN_ARCADE_URL= +STN_CHAINTRACKS_URL= +TSTN_ARCADE_URL= +TSTN_CHAINTRACKS_URL= + # Logging level LOG_LEVEL=info @@ -28,7 +43,8 @@ OTEL_DIAG=false DEPLOY_ENV=development # SOURCE_CDN_URL - Remote CDN to download bulk headers FROM (fallback when local files don't exist) -# This is where the ingestor will fetch headers if they're not in the local filesystem +# This is where the ingestor will fetch headers if they're not in the local filesystem. +# Set an explicit empty value to disable the CDN source. # Example: https://cdn.projectbabbage.com/blockheaders SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ @@ -64,8 +80,13 @@ BULK_HEADERS_AUTO_EXPORT_INTERVAL=240000000 # CHAINTRACKS_STRICT_TRANSPORT_SECURITY=false # API edge policy. +CHAINTRACKS_RESOURCE_PROFILE=standard CHAINTRACKS_MAX_BODY_BYTES=262144 -CHAINTRACKS_MAX_CONCURRENT_REQUESTS=200 +CHAINTRACKS_MAX_RESPONSE_BYTES=4194304 +CHAINTRACKS_MAX_CONCURRENT_REQUESTS=64 +CHAINTRACKS_MAX_CONNECTIONS=1000 +CHAINTRACKS_HEADERS_DEFAULT_LIMIT=1000 +CHAINTRACKS_HEADERS_MAX_LIMIT=1000 CHAINTRACKS_REQUEST_TIMEOUT_MS=60000 CHAINTRACKS_HEADERS_TIMEOUT_MS=15000 CHAINTRACKS_KEEP_ALIVE_TIMEOUT_MS=5000 @@ -74,6 +95,7 @@ CHAINTRACKS_MAX_REQUESTS_PER_SOCKET=1000 # Static header CDN edge policy. CHAINTRACKS_CDN_MAX_CONCURRENT_REQUESTS=100 +CHAINTRACKS_CDN_MAX_CONNECTIONS=1000 CHAINTRACKS_CDN_REQUEST_TIMEOUT_MS=30000 CHAINTRACKS_CDN_HEADERS_TIMEOUT_MS=10000 CHAINTRACKS_CDN_KEEP_ALIVE_TIMEOUT_MS=5000 diff --git a/infra/chaintracks-server/API.md b/infra/chaintracks-server/API.md index b71fae7e2..ce6e6a683 100644 --- a/infra/chaintracks-server/API.md +++ b/infra/chaintracks-server/API.md @@ -51,17 +51,36 @@ Access-Control-Allow-Private-Network: true ## Endpoints +The legacy v1-style routes documented below remain unchanged. The service also +provides this go-chaintracks-compatible v2 surface: + +| Method | Path | Result | +| ------ | ---------------------------------- | ---------------------------- | +| GET | `/v2/network` | Configured network | +| GET | `/v2/height` | Present height | +| GET | `/v2/tip` | Active tip JSON | +| GET | `/v2/tip/stream` | Tip SSE stream | +| GET | `/v2/reorg/stream` | Reorganization SSE stream | +| GET | `/v2/header/height/:height` | Header JSON | +| GET | `/v2/header/hash/:hash` | Header JSON | +| GET | `/v2/headers.bin?height=N&count=M` | Concatenated 80-byte headers | + +JSON responses accept both raw go-chaintracks values and the server's existing +`{status,value}` compatibility envelope through `GoChaintracksServiceClient`. + ### GET / Returns server information page. **Response:** + ``` Content-Type: text/plain Chaintracks mainNet Block Header Service ``` **Example:** + ```bash curl http://localhost:3011/ ``` @@ -73,6 +92,7 @@ curl http://localhost:3011/ Returns robots exclusion standard file. **Response:** + ``` User-agent: * Disallow: / @@ -85,6 +105,7 @@ Disallow: / Returns the blockchain network the service is tracking. **Response:** + ```json { "status": "success", @@ -93,15 +114,21 @@ Returns the blockchain network the service is tracking. ``` **Values:** + - `"main"` - Bitcoin SV mainnet - `"test"` - Bitcoin SV testnet +- `"stn"` - Scaling Test Network +- `"ttn"` - TerraTestNet +- `"tstn"` - Terra Scaling TestNet **Example:** + ```bash curl http://localhost:3011/getChain ``` **Response Example:** + ```json { "status": "success", @@ -116,9 +143,11 @@ curl http://localhost:3011/getChain Returns detailed information about the service state, configuration, and current blockchain heights. **Query Parameters:** + - `wait` (optional): Milliseconds to wait before responding (for testing) **Response:** + ```json { "status": "success", @@ -127,33 +156,36 @@ Returns detailed information about the service state, configuration, and current "heightBulk": 869999, "heightLive": 870125, "storage": "ChaintracksStorageNoDb", - "bulkIngestors": [ - "BulkIngestorCDNBabbage", - "BulkIngestorWhatsOnChainCdn" - ], - "liveIngestors": [ - "LiveIngestorWhatsOnChainPoll" - ], - "packages": [] + "bulkIngestors": ["BulkIngestorCDNBabbage", "BulkIngestorWhatsOnChainCdn"], + "liveIngestors": ["LiveIngestorWhatsOnChainPoll"], + "packages": [], + "sources": [ + { "name": "bulk[1]:BulkIngestorChaintracks", "role": "bulk", "state": "healthy" }, + { "name": "live[0]:LiveIngestorChaintracksSSE", "role": "live", "state": "healthy" } + ] } } ``` **Fields:** -- `chain`: Network name ('main' or 'test') + +- `chain`: Network name (`main`, `test`, `stn`, `ttn`, or `tstn`) - `heightBulk`: Highest height in bulk storage (CDN-backed) - `heightLive`: Highest height in live storage (in-memory) - `storage`: Storage backend class name - `bulkIngestors`: List of bulk ingestor class names - `liveIngestors`: List of live ingestor class names - `packages`: Package version information (optional) +- `sources`: Last observed health, success/failure time, and error for each source **Example:** + ```bash curl http://localhost:3011/getInfo ``` **Notes:** + - Response is never cached (Cache-Control: no-cache) - Use this endpoint for health checks and monitoring - `heightBulk` should be close to `heightLive` (within ~2000 blocks) @@ -165,6 +197,7 @@ curl http://localhost:3011/getInfo Returns the latest blockchain height from configured bulk ingestors. This represents the current "real" blockchain height from external sources. **Response:** + ```json { "status": "success", @@ -173,11 +206,13 @@ Returns the latest blockchain height from configured bulk ingestors. This repres ``` **Example:** + ```bash curl http://localhost:3011/getPresentHeight ``` **Notes:** + - Response is cached for 1 minute - Value is fetched from WhatsOnChain or other bulk ingestors - Response is never cached (Cache-Control: no-cache) @@ -189,6 +224,7 @@ curl http://localhost:3011/getPresentHeight Returns the block hash of the active chain tip. **Response:** + ```json { "status": "success", @@ -197,11 +233,13 @@ Returns the block hash of the active chain tip. ``` **Example:** + ```bash curl http://localhost:3011/findChainTipHashHex ``` **Notes:** + - Response is never cached (Cache-Control: no-cache) - Returns empty string if no headers available @@ -212,6 +250,7 @@ curl http://localhost:3011/findChainTipHashHex Returns the complete block header of the active chain tip. **Response:** + ```json { "status": "success", @@ -229,6 +268,7 @@ Returns the complete block header of the active chain tip. ``` **Fields:** + - `version`: Block version number - `previousHash`: Hash of previous block (hex string) - `merkleRoot`: Merkle root of transactions (hex string) @@ -239,11 +279,13 @@ Returns the complete block header of the active chain tip. - `hash`: Block hash (hex string) **Example:** + ```bash curl http://localhost:3011/findChainTipHeaderHex ``` **Notes:** + - Response is never cached (Cache-Control: no-cache) - All hash fields are hex strings (lowercase) @@ -254,9 +296,11 @@ curl http://localhost:3011/findChainTipHeaderHex Returns the block header for a specific height on the active chain. **Query Parameters:** + - `height` (required): Block height (integer) **Response:** + ```json { "status": "success", @@ -274,6 +318,7 @@ Returns the block header for a specific height on the active chain. ``` If height not found: + ```json { "status": "success", @@ -282,11 +327,13 @@ If height not found: ``` **Example:** + ```bash curl "http://localhost:3011/findHeaderHexForHeight?height=800000" ``` **Notes:** + - Returns `null` if height doesn't exist - Only returns headers on active chain - Fast O(1) lookup @@ -298,9 +345,11 @@ curl "http://localhost:3011/findHeaderHexForHeight?height=800000" Returns the block header for a specific block hash (if in live storage). **Query Parameters:** + - `hash` (required): Block hash (hex string) **Response:** + ```json { "status": "success", @@ -318,6 +367,7 @@ Returns the block header for a specific block hash (if in live storage). ``` If hash not found: + ```json { "status": "success", @@ -326,11 +376,13 @@ If hash not found: ``` **Example:** + ```bash curl "http://localhost:3011/findHeaderHexForBlockHash?hash=00000000000000000123456789abcd..." ``` **Notes:** + - Only searches live storage (recent ~2000 blocks) - Returns `null` if hash not found or in bulk storage - For older headers, use `findHeaderHexForHeight` instead @@ -342,10 +394,12 @@ curl "http://localhost:3011/findHeaderHexForBlockHash?hash=000000000000000001234 Returns multiple block headers in serialized format starting from a specific height. **Query Parameters:** + - `height` (required): Starting block height (integer) - `count` (required): Number of headers to return (integer, max recommended: 1000) **Response:** + ```json { "status": "success", @@ -354,18 +408,21 @@ Returns multiple block headers in serialized format starting from a specific hei ``` **Format:** + - Returns hex string of concatenated 80-byte block headers - Each header is 80 bytes (160 hex characters) - Total length: `count × 160` characters - Headers are in order from `height` to `height + count - 1` **Example:** + ```bash # Get 10 headers starting from height 800000 curl "http://localhost:3011/getHeaders?height=800000&count=10" ``` **Parsing the Response:** + ```javascript const response = await fetch('http://localhost:3011/getHeaders?height=800000&count=10') const data = await response.json() @@ -383,6 +440,7 @@ for (let i = 0; i < count; i++) { ``` **Notes:** + - Efficient for bulk header downloads - Use for SPV client synchronization - Recommended to request in batches (e.g., 100-1000 headers) @@ -394,6 +452,7 @@ for (let i = 0; i < count; i++) { Returns current fiat exchange rates for BSV from configured services. **Response:** + ```json { "status": "success", @@ -411,11 +470,13 @@ Returns current fiat exchange rates for BSV from configured services. ``` **Example:** + ```bash curl http://localhost:3011/getFiatExchangeRates ``` **Notes:** + - Response is never cached (Cache-Control: no-cache) - Rates are fetched from external services - May return empty object if services unavailable @@ -427,6 +488,7 @@ curl http://localhost:3011/getFiatExchangeRates Submits a new block header for consideration and processing. **Request Body:** + ```json { "version": 536870912, @@ -439,6 +501,7 @@ Submits a new block header for consideration and processing. ``` **Response:** + ```json { "status": "success" @@ -446,6 +509,7 @@ Submits a new block header for consideration and processing. ``` **Fields:** + - `version`: Block version number (integer) - `previousHash`: Hash of previous block (hex string, 64 chars) - `merkleRoot`: Merkle root (hex string, 64 chars) @@ -454,6 +518,7 @@ Submits a new block header for consideration and processing. - `nonce`: Block nonce (integer) **Example:** + ```bash curl -X POST http://localhost:3011/addHeaderHex \ -H "Content-Type: application/json" \ @@ -468,12 +533,14 @@ curl -X POST http://localhost:3011/addHeaderHex \ ``` **Processing:** + - Header is queued for processing (returns immediately) - Header is validated and inserted asynchronously - If previous header is unknown, header is ignored - Invalid headers are rejected silently **Notes:** + - Response does not indicate if header was accepted/added - Use for submitting newly mined blocks - Header must have valid proof-of-work @@ -488,6 +555,7 @@ curl -X POST http://localhost:3011/addHeaderHex \ Generic internal server error. **Example:** + ```json { "status": "error", @@ -497,6 +565,7 @@ Generic internal server error. ``` **Common Causes:** + - Storage operation failed - Invalid data format - Unhandled exception @@ -520,6 +589,7 @@ The server does not implement rate limiting by default. For production use, cons Most endpoints include cache headers: **No Cache (dynamic data):** + ``` Cache-Control: no-cache, no-store, must-revalidate Pragma: no-cache @@ -527,6 +597,7 @@ Expires: 0 ``` Applies to: + - `/getInfo` - `/getPresentHeight` - `/findChainTipHashHex` @@ -536,10 +607,12 @@ Applies to: **Cacheable (static data):** No explicit cache headers. Clients may cache based on: + - Block height (immutable once confirmed) - Block hash (immutable) Applies to: + - `/findHeaderHexForHeight?height=N` (for heights < chain tip - 100) - `/findHeaderHexForBlockHash?hash=H` (for deep blocks) - `/getHeaders?height=N&count=M` (for heights < chain tip - 100) @@ -554,13 +627,15 @@ The service implements internal caching: --- -## WebSocket Support +## Real-Time SSE + +The go-chaintracks-compatible v2 API exposes browser-safe server-sent events: -The service does not currently support WebSocket connections. For real-time updates: +- `GET /v2/tip/stream` sends the current tip and subsequent active tips. +- `GET /v2/reorg/stream` sends reorganization depth, old/new tips, and optional deactivated headers. +- Keepalive comments are sent every 15 seconds. Clients should reconnect with bounded backoff; `GoChaintracksServiceClient` does this automatically. -1. Poll `/getInfo` endpoint (recommended interval: 30-60 seconds) -2. Subscribe to events programmatically if using the service as a library -3. Implement custom WebSocket wrapper on top of the service +The service does not expose a WebSocket protocol. --- @@ -569,13 +644,22 @@ The service does not currently support WebSocket connections. For real-time upda ### Basic Health Check ```bash -curl http://localhost:3011/getInfo +curl http://localhost:3011/healthz ``` Check that: + - Response status is 200 -- `status` field is "success" -- `heightLive` is increasing over time +- `status` field is `ok` + +Use readiness for synchronization and source state: + +```bash +curl http://localhost:3011/readyz +``` + +When `ROUTING_PREFIX` is configured, `/readyz` and all protocol routes move +under that prefix; root `/healthz` remains available for process liveness. ### Synchronization Health diff --git a/infra/chaintracks-server/ARCHITECTURE.md b/infra/chaintracks-server/ARCHITECTURE.md index f5486c527..bae246e74 100644 --- a/infra/chaintracks-server/ARCHITECTURE.md +++ b/infra/chaintracks-server/ARCHITECTURE.md @@ -80,14 +80,26 @@ This document provides a deep dive into the ChaintracksService architecture, ini │ │ - Returns bulk files with 100k headers each │ │ │ └──────────────────────────────────────────────────────────┘ │ │ ┌──────────────────────────────────────────────────────────┐ │ +│ │ BulkIngestorChaintracks │ │ +│ │ - Fetches bounded binary batches from Arcade/go v2 │ │ +│ │ - Verifies the configured upstream network │ │ +│ │ - Passes every batch through local chain validation │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────┐ │ │ │ BulkIngestorWhatsOnChainCdn │ │ │ │ - Fetches from WhatsOnChain CDN │ │ -│ │ - Fallback if Babbage CDN unavailable │ │ -│ │ - Uses WoC API key if configured │ │ +│ │ - Mainnet/testnet fallback after CDN and Arcade │ │ +│ │ - Keyless requests remain below the public rate limit │ │ │ └──────────────────────────────────────────────────────────┘ │ │ │ │ Live Ingestors (Real-time Headers): │ │ ┌──────────────────────────────────────────────────────────┐ │ +│ │ LiveIngestorChaintracksSSE │ │ +│ │ - Follows Arcade/go v2 tip events │ │ +│ │ - Reconnects with bounded exponential backoff │ │ +│ │ - Works in Node.js, browser, and mobile runtimes │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────┐ │ │ │ LiveIngestorWhatsOnChainPoll │ │ │ │ - Polls WoC /chain/info endpoint │ │ │ │ - Detects new blocks via height change │ │ @@ -98,6 +110,13 @@ This document provides a deep dive into the ChaintracksService architecture, ini └────────────────────────────────────────────────────────────────┘ ``` +Sources are tried in priority order rather than queried concurrently. Failures +are recorded per source and fall through to the next configured provider. Once +synchronized, the locally validated header range and cached last-good height +remain serviceable during a provider outage. Arcade supplies the HTTP/SSE +gateway to Teranode-backed data; direct P2P is intentionally server-only future +work and is not included in browser/mobile artifacts. + ## Initialization Flow ### 1. ChaintracksService Constructor @@ -355,13 +374,13 @@ class ChaintracksStorageNoDb { // Height ranges private ranges: { - bulk: HeightRange, // {minHeight, maxHeight} - live: HeightRange // {minHeight, maxHeight} + bulk: HeightRange // {minHeight, maxHeight} + live: HeightRange // {minHeight, maxHeight} } // Configuration - liveHeightThreshold: number = 2000 // Headers within this are "live" - reorgHeightThreshold: number = 400 // Max reorg depth to handle + liveHeightThreshold: number = 2000 // Headers within this are "live" + reorgHeightThreshold: number = 400 // Max reorg depth to handle } ``` @@ -393,11 +412,9 @@ class BulkFileDataManager { ```typescript // Subscribe to new headers -const subscriptionId = await chaintracks.subscribeHeaders( - (header: BlockHeader) => { - console.log('New block:', header.height, header.hash) - } -) +const subscriptionId = await chaintracks.subscribeHeaders((header: BlockHeader) => { + console.log('New block:', header.height, header.hash) +}) // Called when: // - Header is added (inserted successfully) @@ -414,7 +431,10 @@ const subscriptionId = await chaintracks.subscribeReorgs( console.log(`Reorg: ${depth} blocks`) console.log('Old tip:', oldTip.hash) console.log('New tip:', newTip.hash) - console.log('Deactivated:', deactivated?.map(h => h.hash)) + console.log( + 'Deactivated:', + deactivated?.map(h => h.hash) + ) } ) diff --git a/infra/chaintracks-server/DOCKER.md b/infra/chaintracks-server/DOCKER.md index 7937a3b21..725718361 100644 --- a/infra/chaintracks-server/DOCKER.md +++ b/infra/chaintracks-server/DOCKER.md @@ -23,6 +23,7 @@ docker compose logs -f ``` That's it! The service is now running with: + - ChaintracksService on http://localhost:3011 - Bulk Headers CDN on http://localhost:3012 @@ -53,10 +54,21 @@ Edit `.env` file to customize: ```bash # Chain selection -CHAIN=main # or 'test' for testnet - -# WhatsOnChain API Key (recommended for better rate limits) -WHATSONCHAIN_API_KEY=your_api_key_here +CHAIN=main # main | test | stn | ttn | tstn + +# Optional. Anonymous main/test fallback is already rate limited. +WHATSONCHAIN_API_KEY= + +# Public credential-free defaults exist for main/test/ttn. Configure a v2 +# Arcade/go-chaintracks endpoint for stn/tstn. +CHAINTRACKS_UPSTREAM_URL= +CHAINTRACKS_UPSTREAM_API_PREFIX= +CHAINTRACKS_UPSTREAM_MAX_HEADERS=1000 +CHAINTRACKS_DISABLE_WHATSONCHAIN=false +STN_ARCADE_URL= +STN_CHAINTRACKS_URL= +TSTN_ARCADE_URL= +TSTN_CHAINTRACKS_URL= # Source CDN (where to download FROM if local files don't exist) SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ @@ -77,11 +89,13 @@ BULK_HEADERS_AUTO_EXPORT_INTERVAL=240000000 For production deployment: 1. **Set CDN_HOST_URL to your domain:** + ```bash CDN_HOST_URL=https://headers.yourdomain.com ``` 2. **Configure reverse proxy (nginx example):** + ```nginx # Proxy to CDN server server { @@ -113,11 +127,13 @@ For production deployment: ## Docker Commands ### Start the service + ```bash docker compose up -d ``` ### View logs + ```bash # All logs docker compose logs -f @@ -127,21 +143,25 @@ docker compose logs -f chaintracks-server ``` ### Stop the service + ```bash docker compose down ``` ### Restart the service + ```bash docker compose restart ``` ### Rebuild after code changes + ```bash docker compose up -d --build ``` ### View resource usage + ```bash docker stats chaintracks-server ``` @@ -149,6 +169,7 @@ docker stats chaintracks-server ## Volumes ### Viewing bulk headers + ```bash # List files docker compose exec chaintracks-server ls -lh /app/public/headers @@ -158,6 +179,7 @@ docker compose exec chaintracks-server cat /app/public/headers/mainNetBlockHeade ``` ### Backup bulk headers + ```bash # Create backup docker run --rm -v chaintracks-server_bulk-headers:/data -v $(pwd):/backup alpine tar czf /backup/headers-backup.tar.gz -C /data . @@ -167,6 +189,7 @@ docker run --rm -v chaintracks-server_bulk-headers:/data -v $(pwd):/backup alpin ``` ### Clean up volumes + ```bash # Stop and remove containers and volumes docker compose down -v @@ -175,6 +198,7 @@ docker compose down -v ## Accessing the Services ### ChaintracksService API (Port 3011) + ```bash # Get chain info curl http://localhost:3011/getInfo @@ -187,6 +211,7 @@ curl http://localhost:3011/findChainTipHeader ``` ### Bulk Headers CDN (Port 3012) + ```bash # Get metadata curl http://localhost:3012/mainNetBlockHeaders.json @@ -201,6 +226,7 @@ ls -lh mainNet_0.headers ## Troubleshooting ### Container won't start + ```bash # Check logs for errors docker compose logs chaintracks-server @@ -211,6 +237,7 @@ lsof -i :3012 ``` ### Out of disk space + ```bash # Check volume size docker system df -v @@ -220,6 +247,7 @@ docker system prune -a ``` ### Headers not exporting + ```bash # Check logs for export messages docker compose logs chaintracks-server | grep -i export @@ -232,6 +260,7 @@ docker compose restart chaintracks-server ``` ### Slow sync + - Add `WHATSONCHAIN_API_KEY` for better rate limits - Increase `SOURCE_CDN_URL` if you have a closer CDN - Check resource limits in `docker-compose.yml` @@ -239,11 +268,13 @@ docker compose restart chaintracks-server ## Resource Requirements **Minimum:** + - CPU: 1 core - RAM: 2 GB - Disk: 5 GB (for headers) **Recommended:** + - CPU: 2 cores - RAM: 4 GB - Disk: 10 GB (with room to grow) @@ -251,6 +282,7 @@ docker compose restart chaintracks-server ## Monitoring ### Health Check + Docker Compose includes a health check that verifies the service is responding: ```bash @@ -259,6 +291,7 @@ docker compose ps ``` ### Logs + ```bash # Follow logs docker compose logs -f @@ -288,6 +321,7 @@ docker compose logs -f To have other servers use YOUR server as a CDN source: **On other servers, set:** + ```bash SOURCE_CDN_URL=http://yourserver:3012 # or @@ -323,6 +357,7 @@ networks: ## Support For issues: + 1. Check logs: `docker compose logs -f` 2. Check GitHub issues 3. Verify configuration in `.env` diff --git a/infra/chaintracks-server/Dockerfile b/infra/chaintracks-server/Dockerfile index 029b5bcb6..6ea500030 100644 --- a/infra/chaintracks-server/Dockerfile +++ b/infra/chaintracks-server/Dockerfile @@ -59,7 +59,7 @@ USER node EXPOSE 3011 3012 HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ - CMD ["node", "-e", "const port=process.env.PORT||'3011';fetch('http://127.0.0.1:'+port+'/getInfo').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.PORT||'3011';const prefix=(process.env.ROUTING_PREFIX||'').replace(/\\/+$/,'');fetch('http://127.0.0.1:'+port+prefix+'/readyz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] # Run the application with the OpenTelemetry bootstrap preloaded so # auto-instrumentation patches modules before app code is imported. diff --git a/infra/chaintracks-server/README.md b/infra/chaintracks-server/README.md index ab632742b..85078b83d 100644 --- a/infra/chaintracks-server/README.md +++ b/infra/chaintracks-server/README.md @@ -2,6 +2,14 @@ A production-ready TypeScript Express server wrapping `ChaintracksService` from `@bsv/wallet-toolbox`, featuring a built-in **Bulk Headers CDN** for hosting and serving blockchain headers to other servers. +ChainTracks supports mainnet, testnet, STN, TerraTestNet (`ttn`), and Terra +Scaling TestNet (`tstn`). Mainnet, testnet, and TTN use public credential-free +Arcade/go-chaintracks v2 sources by default. STN and TSTN require an explicit +operator endpoint and are never silently mapped to testnet. + +Resource profiles, limits, memory evidence, and scaling guidance are documented +in [Service Resource Profiles](../../docs/reference/service-resource-profiles.md). + ## 🚀 Quick Start ### Docker (Recommended) @@ -41,6 +49,7 @@ npm start This server provides two main services: ### 1. ChaintracksService (Port 3011) + - **Tracks BSV blockchain headers** in real-time - **In-memory NoDb storage** - no database required - **REST API endpoints** for querying headers @@ -48,6 +57,7 @@ This server provides two main services: - **Event subscriptions** for headers and reorgs ### 2. Bulk Headers CDN (Port 3012) + - **Hosts bulk header files** for download by other servers - **Automatic export** at 100k block boundaries - **Self-hosting CDN** - becomes a headers source for others @@ -57,13 +67,16 @@ This server provides two main services: ## ✨ Key Features ### 🌐 Self-Hosting CDN Network + Your server can become a CDN node: + 1. Downloads headers from remote CDN (if local files don't exist) 2. Exports headers to filesystem 3. Serves headers to other servers via HTTP 4. Creates a distributed network of header sources ### 📦 Automatic Header Management + - Downloads from `SOURCE_CDN_URL` on first startup - Exports to filesystem automatically - Serves via CDN on port 3012 @@ -71,10 +84,23 @@ Your server can become a CDN node: - Triggers export at 100k boundaries ### 🔄 Zero-Config Synchronization -- First run: Downloads from remote CDN + +- Mainnet/testnet: uses the CDN plus public Arcade binary and SSE APIs +- TTN: uses the public Arcade binary and SSE APIs - Subsequent runs: Uses local filesystem - Automatically exports new headers - Other servers can use you as a source +- WhatsOnChain is a mainnet/testnet fallback and does not require a key + +Remote headers pass through local serialization, hash, continuity, and genesis +checks before storage. Source failures fall through in priority order, SSE +reconnects with bounded backoff, and a synchronized process keeps serving +last-good checked data while reporting degraded sources from `/getInfo` and +`/readyz`. + +Arcade is the HTTPS/SSE gateway suitable for Node.js, browsers, mobile clients, +and local services. It may be backed by Teranode P2P. Direct Teranode P2P is not +bundled into this TypeScript/browser distribution. ## 🎯 Architecture @@ -112,11 +138,14 @@ Your server can become a CDN node: All endpoints return JSON with `{ status: "success", value: }` or `{ status: "error", code: "...", description: "..." }` #### Chain Information -- `GET /getChain` - Get blockchain network ('main' or 'test') + +- `GET /getChain` - Get blockchain network (`main`, `test`, `stn`, `ttn`, or `tstn`) - `GET /getInfo` - Detailed service information - `GET /getPresentHeight` - Latest available height +- `GET /readyz` - Readiness, height, and source-health state #### Header Queries + - `GET /findChainTipHeaderHex` - Get chain tip header as hex - `GET /findChainTipHashHex` - Get chain tip hash as hex - `GET /findHeaderHexForHeight?height=N` - Get header at height N as hex @@ -125,6 +154,7 @@ All endpoints return JSON with `{ status: "success", value: }` or `{ stat - `POST /addHeaderHex` - Submit a new block header (JSON body with version, previousHash, merkleRoot, time, bits, nonce) **Note:** The `findHeaderHexForBlockHash` endpoint only works for headers currently retained in memory: + - Recent headers within ~2,000 blocks of chain tip ("live" headers) - Headers in the most recently retained bulk files (~200k headers with default `maxRetained: 2`) - For querying arbitrary historical headers, use `findHeaderHexForHeight?height=N` instead @@ -148,13 +178,24 @@ Create `.env` file (copy from `.env.example`): ```bash # Chain selection -CHAIN=main # or 'test' +CHAIN=main # main | test | stn | ttn | tstn # Server port (ChaintracksService) PORT=3011 -# WhatsOnChain API Key (recommended for production) -WHATSONCHAIN_API_KEY=your_api_key_here +# Optional. Anonymous fallback is capped below the documented 3 requests/sec. +WHATSONCHAIN_API_KEY= + +# Optional go-chaintracks v2 override. Public defaults exist for main/test/ttn. +# Required for stn/tstn unless the matching Arcade/ChainTracks URL is set. +CHAINTRACKS_UPSTREAM_URL= +CHAINTRACKS_UPSTREAM_API_PREFIX= +CHAINTRACKS_UPSTREAM_MAX_HEADERS=1000 +CHAINTRACKS_DISABLE_WHATSONCHAIN=false +STN_ARCADE_URL= +STN_CHAINTRACKS_URL= +TSTN_ARCADE_URL= +TSTN_CHAINTRACKS_URL= # SOURCE_CDN_URL - Where to download headers FROM (if local files don't exist) SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ @@ -181,7 +222,7 @@ BULK_HEADERS_AUTO_EXPORT_INTERVAL=240000000 ```bash CHAIN=main PORT=3011 -WHATSONCHAIN_API_KEY=your_api_key +WHATSONCHAIN_API_KEY= ENABLE_BULK_HEADERS_CDN=true CDN_HOST_URL=https://headers.yourdomain.com SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ @@ -238,6 +279,7 @@ See [DOCKER.md](DOCKER.md) for comprehensive Docker documentation. ## 📚 How It Works ### First Startup + 1. Server starts and checks `./public/headers` for existing files 2. No files found, downloads from `SOURCE_CDN_URL` 3. Syncs blockchain headers to current height @@ -245,12 +287,14 @@ See [DOCKER.md](DOCKER.md) for comprehensive Docker documentation. 5. CDN server starts serving files on port 3012 ### Subsequent Startups + 1. Server starts and checks `./public/headers` 2. Finds existing files, loads them directly (no download!) 3. Continues syncing from last height 4. Automatically exports new headers at 100k boundaries ### Becoming a CDN Source + Other servers can now point to YOUR server: ```bash @@ -263,6 +307,7 @@ This creates a **distributed CDN network** where servers help each other! ## 📦 File Structure ### Bulk Headers Directory + ``` public/headers/ ├── mainNetBlockHeaders.json # Metadata with file list @@ -273,6 +318,7 @@ public/headers/ ``` ### JSON Metadata Format + ```json { "rootFolder": "https://headers.yourdomain.com", @@ -295,11 +341,13 @@ public/headers/ ## 🔧 Development ### Build + ```bash npm run build ``` ### Run Different Configurations + ```bash # Standard server (port 3011) npm start @@ -312,6 +360,7 @@ npm run dev ``` ### Project Structure + ``` ├── src/ │ ├── server.ts # Main server with CDN @@ -343,6 +392,7 @@ SOURCE_CDN_URL=https://headers.yourdomain.com ### Distributed Network Example **Server A (Public CDN):** + ```bash ENABLE_BULK_HEADERS_CDN=true CDN_HOST_URL=https://cdn.example.com @@ -350,6 +400,7 @@ SOURCE_CDN_URL=https://cdn.projectbabbage.com/blockheaders/ ``` **Server B (Uses Server A):** + ```bash ENABLE_BULK_HEADERS_CDN=true CDN_HOST_URL=https://headers-b.example.com @@ -357,6 +408,7 @@ SOURCE_CDN_URL=https://cdn.example.com # Points to Server A ``` **Server C (Uses Server B):** + ```bash ENABLE_BULK_HEADERS_CDN=true CDN_HOST_URL=https://headers-c.example.com @@ -368,16 +420,19 @@ Creates a **self-healing, distributed CDN network**! 🌍 ## 📊 Resource Requirements ### Minimum + - **CPU:** 1 core - **RAM:** 2 GB - **Disk:** 5 GB (for headers) ### Recommended + - **CPU:** 2 cores - **RAM:** 4 GB - **Disk:** 10 GB (with growth room) ### Storage Growth + - ~7.6 MB per 100k blocks - Current blockchain: ~920k blocks = ~70 MB - Growth: ~7.6 MB per ~67 days (at 10 min blocks) @@ -385,6 +440,7 @@ Creates a **self-healing, distributed CDN network**! 🌍 ## 🔍 Monitoring ### Check Service Status + ```bash # API health curl http://localhost:3011/getInfo @@ -394,11 +450,13 @@ curl http://localhost:3012/mainNetBlockHeaders.json ``` ### View Logs (Docker) + ```bash docker compose logs -f ``` ### View Exported Files + ```bash ls -lh public/headers/ ``` @@ -406,22 +464,30 @@ ls -lh public/headers/ ## 🆘 Troubleshooting ### Headers Not Exporting + - Check `ENABLE_BULK_HEADERS_CDN=true` in `.env` - Check logs for export messages - Verify disk space available - Restart server to trigger export ### CDN Files Not Accessible + - Verify CDN server running on port 3012 - Check firewall rules - Test locally: `curl http://localhost:3012/mainNetBlockHeaders.json` ### Slow Sync -- Add `WHATSONCHAIN_API_KEY` for better rate limits + +- Check the configured Arcade/go-chaintracks source and `/readyz` source states - Check `SOURCE_CDN_URL` is reachable - Verify network connectivity +The service does not need a WhatsOnChain key. If one is configured and rejected, +ChainTracks retries anonymously; remove stale keys unless higher paid limits are +actually needed. + ### Docker Issues + See [DOCKER.md](DOCKER.md) troubleshooting section. ## 📖 Additional Documentation @@ -433,6 +499,7 @@ See [DOCKER.md](DOCKER.md) troubleshooting section. ## 🤝 Contributing Contributions welcome! Please: + 1. Fork the repository 2. Create a feature branch 3. Make your changes diff --git a/infra/chaintracks-server/docker-compose.yml b/infra/chaintracks-server/docker-compose.yml index e59f56493..8067fc4a3 100644 --- a/infra/chaintracks-server/docker-compose.yml +++ b/infra/chaintracks-server/docker-compose.yml @@ -21,19 +21,30 @@ services: container_name: chaintracks-server restart: unless-stopped ports: - - "3011:3011" # ChaintracksService - - "3012:3012" # CDN Server + - '3011:3011' # ChaintracksService + - '3012:3012' # CDN Server environment: # Chain configuration - - CHAIN=main + - CHAIN=${CHAIN:-main} - PORT=3011 - # WhatsOnChain API Key (optional but recommended) - # Get your key at: https://whatsonchain.com/ + # Optional higher-limit key; anonymous fallback works without it. - WHATSONCHAIN_API_KEY=${WHATSONCHAIN_API_KEY:-} + # Arcade/go-chaintracks source. Public defaults exist for main/test/ttn. + - CHAINTRACKS_UPSTREAM_URL=${CHAINTRACKS_UPSTREAM_URL:-} + - CHAINTRACKS_UPSTREAM_API_PREFIX=${CHAINTRACKS_UPSTREAM_API_PREFIX:-} + - CHAINTRACKS_UPSTREAM_MAX_HEADERS=${CHAINTRACKS_UPSTREAM_MAX_HEADERS:-1000} + - CHAINTRACKS_DISABLE_WHATSONCHAIN=${CHAINTRACKS_DISABLE_WHATSONCHAIN:-false} + - STN_ARCADE_URL=${STN_ARCADE_URL:-} + - STN_CHAINTRACKS_URL=${STN_CHAINTRACKS_URL:-} + - TSTN_ARCADE_URL=${TSTN_ARCADE_URL:-} + - TSTN_CHAINTRACKS_URL=${TSTN_CHAINTRACKS_URL:-} + - ROUTING_PREFIX=${ROUTING_PREFIX:-} + # Source CDN - Where to download headers FROM if local files don't exist - - SOURCE_CDN_URL=${SOURCE_CDN_URL:-https://cdn.projectbabbage.com/blockheaders/} + # Use `${SOURCE_CDN_URL-...}` so an explicitly empty value disables it. + - SOURCE_CDN_URL=${SOURCE_CDN_URL-https://cdn.projectbabbage.com/blockheaders/} # Bulk Headers CDN - Enable hosting - ENABLE_BULK_HEADERS_CDN=${ENABLE_BULK_HEADERS_CDN:-true} @@ -85,7 +96,11 @@ services: - bulk-headers:/app/public/headers healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:3011/"] + test: + [ + 'CMD-SHELL', + 'wget --quiet --tries=1 --spider "http://localhost:3011$${ROUTING_PREFIX:-}/readyz"' + ] interval: 30s timeout: 10s retries: 3 diff --git a/infra/chaintracks-server/package-lock.json b/infra/chaintracks-server/package-lock.json index e51e68c6b..3422cb286 100644 --- a/infra/chaintracks-server/package-lock.json +++ b/infra/chaintracks-server/package-lock.json @@ -1,12 +1,12 @@ { "name": "chaintracks-server", - "version": "1.0.17", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "chaintracks-server", - "version": "1.0.17", + "version": "1.1.0", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/wallet-toolbox": "^2.5.0", diff --git a/infra/chaintracks-server/package.json b/infra/chaintracks-server/package.json index aea7d9940..2ea46c069 100644 --- a/infra/chaintracks-server/package.json +++ b/infra/chaintracks-server/package.json @@ -1,6 +1,6 @@ { "name": "chaintracks-server", - "version": "1.0.17", + "version": "1.1.0", "overrides": { "gaxios": "7.3.0" }, diff --git a/infra/chaintracks-server/src/resourceLimits.ts b/infra/chaintracks-server/src/resourceLimits.ts new file mode 100644 index 000000000..9f6163212 --- /dev/null +++ b/infra/chaintracks-server/src/resourceLimits.ts @@ -0,0 +1,49 @@ +import { profileValue, readResourceLimit, readResourceProfile } from './security/edgePolicy' + +export interface HeaderRange { + height: number + count: number +} + +export function parseHeaderRange(query: Record): HeaderRange { + const height = Number(query.height) + const profile = readResourceProfile('CHAINTRACKS') + const configuredDefault = readResourceLimit( + 'CHAINTRACKS', + 'HEADERS_DEFAULT_LIMIT', + profileValue(profile, { small: 250, standard: 1_000, highThroughput: 5_000 }) + ) + const configuredMaximum = readResourceLimit( + 'CHAINTRACKS', + 'HEADERS_MAX_LIMIT', + profileValue(profile, { small: 500, standard: 1_000, highThroughput: 5_000 }) + ) + if ( + configuredDefault !== -1 && + configuredMaximum !== -1 && + configuredDefault > configuredMaximum + ) { + throw new Error( + 'CHAINTRACKS_HEADERS_DEFAULT_LIMIT must not exceed CHAINTRACKS_HEADERS_MAX_LIMIT' + ) + } + let count = Number(query.count) + if (query.count == null) { + count = configuredDefault === -1 ? Number.MAX_SAFE_INTEGER : configuredDefault + } + if (!Number.isSafeInteger(height) || height < 0) { + throw new RangeError('Invalid or missing height parameter') + } + if ( + !Number.isSafeInteger(count) || + count < 1 || + (configuredMaximum !== -1 && count > configuredMaximum) + ) { + throw new RangeError( + configuredMaximum === -1 + ? 'count must be a positive safe integer' + : `count must be an integer between 1 and ${configuredMaximum}` + ) + } + return { height, count } +} diff --git a/infra/chaintracks-server/src/security/edgePolicy.ts b/infra/chaintracks-server/src/security/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/infra/chaintracks-server/src/security/edgePolicy.ts +++ b/infra/chaintracks-server/src/security/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/infra/chaintracks-server/src/server.ts b/infra/chaintracks-server/src/server.ts index 34662f2fa..beb4cfd36 100644 --- a/infra/chaintracks-server/src/server.ts +++ b/infra/chaintracks-server/src/server.ts @@ -10,7 +10,16 @@ * - V1 and V2 API routes */ -import { BlockHeader, Chaintracks, createDefaultNoDbChaintracksOptions, Services, Chain, ChaintracksFs } from '@bsv/wallet-toolbox' +import { + BlockHeader, + Chaintracks, + createDefaultNoDbChaintracksOptions, + Services, + Chain, + ChaintracksFs, + GoChaintracksServiceClient +} from '@bsv/wallet-toolbox' +import * as WalletToolbox from '@bsv/wallet-toolbox' import * as path from 'node:path' import * as express from 'express' import * as bodyParser from 'body-parser' @@ -23,40 +32,157 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceProfile, + responseSizeLimit, securityHeaders } from './security/edgePolicy' const tracer = trace.getTracer('chaintracks-server') +type ConfiguredChain = 'main' | 'test' | 'stn' | 'ttn' | 'tstn' + +const supportedChains = new Set(['main', 'test', 'stn', 'ttn', 'tstn']) + +function resolveChain(): ConfiguredChain { + const configured = (process.env.CHAIN || 'main').trim() as ConfiguredChain + if (!supportedChains.has(configured)) { + throw new Error('CHAIN must be "main", "test", "stn", "ttn", or "tstn"') + } + return configured +} + +function stripTrailingSlash(value: string): string { + let end = value.length + while (end > 0 && value[end - 1] === '/') end-- + return value.slice(0, end) +} + +function defaultArcadeUrl(chain: ConfiguredChain): string | undefined { + switch (chain) { + case 'main': + return 'https://arcade-v2-us-1.bsvblockchain.tech' + case 'test': + return 'https://arcade-v2-testnet-us-1.bsvblockchain.tech' + case 'ttn': + return 'https://arcade-v2-ttn-us-1.bsvblockchain.tech' + case 'stn': + return process.env.STN_ARCADE_URL?.trim() || process.env.STN_CHAINTRACKS_URL?.trim() + case 'tstn': + return process.env.TSTN_ARCADE_URL?.trim() || process.env.TSTN_CHAINTRACKS_URL?.trim() + } +} + +function resolveUpstreamChaintracks( + chain: ConfiguredChain +): GoChaintracksServiceClient | undefined { + const configured = process.env.CHAINTRACKS_UPSTREAM_URL?.trim() + if (configured === 'disabled' || configured === 'none') return undefined + const serviceUrl = stripTrailingSlash(configured || defaultArcadeUrl(chain) || '') + if (serviceUrl === '') return undefined + if (/\/v1$/i.test(new URL(serviceUrl).pathname)) { + throw new Error( + 'CHAINTRACKS_UPSTREAM_URL must expose go-chaintracks v2; legacy v1 has no SSE stream.' + ) + } + + let apiPrefix = process.env.CHAINTRACKS_UPSTREAM_API_PREFIX?.trim() + if (apiPrefix == null || apiPrefix === '') { + apiPrefix = /\/v2$/i.test(new URL(serviceUrl).pathname) ? '' : '/chaintracks/v2' + } + return new GoChaintracksServiceClient(chain as Chain, serviceUrl, { + apiPrefix + }) +} + +function resolveRoutingPrefix(): string { + const value = (process.env.ROUTING_PREFIX || '').trim() + if (value === '' || value === '/') return '' + if ( + !value.startsWith('/') || + value.includes('..') || + value.includes('?') || + value.includes('#') + ) { + throw new Error( + 'ROUTING_PREFIX must be an absolute URL path without query, fragment, or parent traversal.' + ) + } + return stripTrailingSlash(value) +} + function resolveBulkHeadersPath(): string { const raw = process.env.BULK_HEADERS_PATH || path.join(process.cwd(), 'public', 'headers') return path.isAbsolute(raw) ? raw : path.join(process.cwd(), raw) } +function createServices(chain: ConfiguredChain, chaintracks: Chaintracks): Services | undefined { + // Keep the standalone source buildable against the currently published + // toolbox while adopting the additive 2.6 factory immediately after the + // protected dependency-lock reconciliation. + const factory = ( + WalletToolbox as unknown as { + createDefaultWalletServicesOptions?: ( + chain: Chain, + ...options: unknown[] + ) => ConstructorParameters[0] + } + ).createDefaultWalletServicesOptions + if (factory != null) { + return new Services( + factory( + chain as Chain, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + chaintracks + ) + ) + } + // 2.5 predates STN in the public Chain union. Other pre-2.6 deployments + // retain their exact Services construction until the lock is reconciled. + return chain === 'stn' ? undefined : new Services(chain as Chain) +} + async function ensureBulkHeadersDir(bulkHeadersPath: string): Promise { try { const fs = await import('node:fs/promises') await fs.mkdir(bulkHeadersPath, { recursive: true }) - log.info({ operation: 'bulk_headers.dir_ensure', outcome: 'ok', bulk_headers_path: bulkHeadersPath }, 'Bulk headers directory ready') + log.info( + { operation: 'bulk_headers.dir_ensure', outcome: 'ok', bulk_headers_path: bulkHeadersPath }, + 'Bulk headers directory ready' + ) } catch (error) { - log.error({ operation: 'bulk_headers.dir_ensure', outcome: 'error', bulk_headers_path: bulkHeadersPath, err: error }, 'Failed to create bulk headers directory') + log.error( + { + operation: 'bulk_headers.dir_ensure', + outcome: 'error', + bulk_headers_path: bulkHeadersPath, + err: error + }, + 'Failed to create bulk headers directory' + ) throw error } } async function main() { - const configuredChain = process.env.CHAIN || 'main' - if (configuredChain !== 'main' && configuredChain !== 'test') { - throw new Error('CHAIN must be "main" or "test"') - } - const chain: Chain = configuredChain + const chain = resolveChain() const port = Number.parseInt(process.env.PORT || '3013', 10) const cdnPort = port + 1 // CDN runs on next port const whatsonchainApiKey = process.env.WHATSONCHAIN_API_KEY || '' // SOURCE_CDN_URL: Remote CDN to download headers FROM (if local files don't exist) - const sourceCdnUrl = process.env.SOURCE_CDN_URL || '' + const defaultSourceCdnUrl = + chain === 'main' || chain === 'test' ? 'https://cdn.projectbabbage.com/blockheaders/' : '' + const sourceCdnUrl = process.env.SOURCE_CDN_URL ?? defaultSourceCdnUrl + const upstreamChaintracks = resolveUpstreamChaintracks(chain) + const routingPrefix = resolveRoutingPrefix() const enableBulkHeadersCDN = process.env.ENABLE_BULK_HEADERS_CDN === 'true' @@ -68,7 +194,10 @@ async function main() { // The source URL is where clients can download headers from (the CDN HTTP endpoint) const bulkHeadersSourceUrl = enableBulkHeadersCDN ? cdnHostUrl : undefined - const bulkHeadersAutoExportInterval = Number.parseInt(process.env.BULK_HEADERS_AUTO_EXPORT_INTERVAL || '240000000', 10) // Default: 400 blocks around 67 hours + const bulkHeadersAutoExportInterval = Number.parseInt( + process.env.BULK_HEADERS_AUTO_EXPORT_INTERVAL || '240000000', + 10 + ) // Default: 400 blocks around 67 hours log.info( { @@ -76,7 +205,10 @@ async function main() { chain: `${chain}Net`, port, whatsonchain_api_key_configured: Boolean(whatsonchainApiKey), - bulk_headers_cdn_enabled: enableBulkHeadersCDN, + whatsonchain_fallback_enabled: chain === 'main' || chain === 'test', + upstream_chaintracks_configured: upstreamChaintracks != null, + routing_prefix: routingPrefix || '/', + bulk_headers_cdn_enabled: enableBulkHeadersCDN }, 'Starting ChaintracksService with custom configuration' ) @@ -86,7 +218,7 @@ async function main() { operation: 'config.cdn', cdn_port: cdnPort, cdn_host_url: cdnHostUrl, - bulk_headers_path: bulkHeadersPath, + bulk_headers_path: bulkHeadersPath }, 'Bulk headers CDN configuration' ) @@ -96,8 +228,14 @@ async function main() { // Create custom Chaintracks options // This allows fine-tuning of storage, ingestors, and sync behavior // When bulk headers CDN is enabled, configure the CDN ingestor to use the local filesystem first - const chaintracksOptions = createDefaultNoDbChaintracksOptions( - chain, + // The standalone image intentionally compiles against the currently + // published toolbox. The appended source argument is consumed by 2.6.0 + // after the protected package release and lock reconciliation. + const createOptions = createDefaultNoDbChaintracksOptions as unknown as ( + ...args: unknown[] + ) => ReturnType + const chaintracksOptions = createOptions( + chain as Chain, whatsonchainApiKey, // WhatsOnChain API key for better rate limits 100000, // maxPerFile: Headers per bulk file (100k) 2, // maxRetained: Number of bulk files to retain in memory @@ -107,7 +245,17 @@ async function main() { 400, // reorgHeightThreshold: Max reorg depth to handle 500, // bulkMigrationChunkSize: Batch size for migrations 400, // batchInsertLimit: Max headers to insert in one batch - 36 // addLiveRecursionLimit: Max depth to recursively fetch missing headers + 36, // addLiveRecursionLimit: Max depth to recursively fetch missing headers + { + chaintracks: upstreamChaintracks, + disableChaintracks: upstreamChaintracks == null, + remoteMaxHeadersPerRequest: Number.parseInt( + process.env.CHAINTRACKS_UPSTREAM_MAX_HEADERS || '1000', + 10 + ), + disableCdn: sourceCdnUrl === '', + disableWhatsOnChain: process.env.CHAINTRACKS_DISABLE_WHATSONCHAIN === 'true' + } ) // If bulk headers CDN is enabled, configure the CDN ingestor to use our local path @@ -124,7 +272,10 @@ async function main() { if (cdnIngestor?.localCachePath !== undefined) { // Override the local cache path to use our bulk headers export directory cdnIngestor.localCachePath = bulkHeadersPath - log.info({ operation: 'cdn.ingestor_configure', outcome: 'ok', local_cache_path: bulkHeadersPath }, 'Configured CDN ingestor to use local path; filesystem checked first, then remote CDN') + log.info( + { operation: 'cdn.ingestor_configure', outcome: 'ok', local_cache_path: bulkHeadersPath }, + 'Configured CDN ingestor to use local path; filesystem checked first, then remote CDN' + ) } } @@ -138,12 +289,18 @@ async function main() { // Function to export bulk headers const exportBulkHeaders = async () => { if (!enableBulkHeadersCDN) { - log.info({ operation: 'headers.export', outcome: 'skipped', reason: 'cdn_disabled' }, 'Bulk headers CDN is disabled, skipping export') + log.info( + { operation: 'headers.export', outcome: 'skipped', reason: 'cdn_disabled' }, + 'Bulk headers CDN is disabled, skipping export' + ) return } if (isExporting) { - log.info({ operation: 'headers.export', outcome: 'skipped', reason: 'in_progress' }, 'Export already in progress, skipping') + log.info( + { operation: 'headers.export', outcome: 'skipped', reason: 'in_progress' }, + 'Export already in progress, skipping' + ) return } @@ -165,7 +322,7 @@ async function main() { last_exported_height: lastExportedHeight, current_milestone: currentMilestone, last_milestone: lastMilestone, - should_export: shouldExport, + should_export: shouldExport }, 'Evaluated export need' ) @@ -175,7 +332,7 @@ async function main() { { operation: 'headers.export', bulk_headers_path: bulkHeadersPath, - source_url: bulkHeadersSourceUrl, + source_url: bulkHeadersSourceUrl }, 'Exporting bulk headers' ) @@ -184,8 +341,8 @@ async function main() { bulkHeadersPath, ChaintracksFs, bulkHeadersSourceUrl, // sourceUrl - sets rootFolder in the JSON metadata file - 100000, // headersPerFile - undefined // maxHeight (export all available) + 100000, // headersPerFile + undefined // maxHeight (export all available) ) lastExportedHeight = currentHeight @@ -194,7 +351,7 @@ async function main() { operation: 'headers.export', outcome: 'ok', bulk_headers_path: bulkHeadersPath, - download_url: `${bulkHeadersSourceUrl}/${chain}NetBlockHeaders.json`, + download_url: `${bulkHeadersSourceUrl}/${chain}NetBlockHeaders.json` }, 'Bulk headers exported successfully' ) @@ -203,15 +360,27 @@ async function main() { const fs = await import('node:fs/promises') try { const files = await fs.readdir(bulkHeadersPath) - log.info({ operation: 'headers.export', file_count: files.length, files }, 'Listed exported files') + log.info( + { operation: 'headers.export', file_count: files.length, files }, + 'Listed exported files' + ) } catch (e) { - log.warn({ operation: 'headers.export', outcome: 'error', err: e }, 'Could not list files') + log.warn( + { operation: 'headers.export', outcome: 'error', err: e }, + 'Could not list files' + ) } } else { - log.info({ operation: 'headers.export', outcome: 'skipped', reason: 'no_boundary_crossed' }, 'No export needed') + log.info( + { operation: 'headers.export', outcome: 'skipped', reason: 'no_boundary_crossed' }, + 'No export needed' + ) } } catch (error) { - log.error({ operation: 'headers.export', outcome: 'error', err: error }, 'Error exporting bulk headers') + log.error( + { operation: 'headers.export', outcome: 'error', err: error }, + 'Error exporting bulk headers' + ) } finally { isExporting = false } @@ -219,31 +388,37 @@ async function main() { // Subscribe to new block header events // This allows you to react to new blocks in real-time - const headerSubscriptionId = await chaintracks.subscribeHeaders( - async (header: BlockHeader) => { - log.info( - { - operation: 'header.received', - height: header.height, - hash: header.hash, - timestamp: new Date(header.time * 1000).toISOString(), - }, - 'New block header received' - ) + const headerSubscriptionId = await chaintracks.subscribeHeaders(async (header: BlockHeader) => { + log.info( + { + operation: 'header.received', + height: header.height, + hash: header.hash, + timestamp: new Date(header.time * 1000).toISOString() + }, + 'New block header received' + ) - // Check if we should export headers (non-blocking) - if (enableBulkHeadersCDN) { - exportBulkHeaders().catch(err => - log.error({ operation: 'headers.export', outcome: 'error', context: 'background', err }, 'Background export error') + // Check if we should export headers (non-blocking) + if (enableBulkHeadersCDN) { + exportBulkHeaders().catch(err => + log.error( + { operation: 'headers.export', outcome: 'error', context: 'background', err }, + 'Background export error' ) - } + ) } - ) + }) // Subscribe to blockchain reorganization events // Important for handling chain reorgs properly const reorgSubscriptionId = await chaintracks.subscribeReorgs( - async (depth: number, oldTip: BlockHeader, newTip: BlockHeader, deactivated?: BlockHeader[]) => { + async ( + depth: number, + oldTip: BlockHeader, + newTip: BlockHeader, + deactivated?: BlockHeader[] + ) => { log.info( { operation: 'reorg.detected', @@ -252,39 +427,97 @@ async function main() { old_tip_height: oldTip.height, new_tip_hash: newTip.hash, new_tip_height: newTip.height, - deactivated_hashes: deactivated && deactivated.length > 0 ? deactivated.map(h => h.hash) : [], + deactivated_hashes: + deactivated && deactivated.length > 0 ? deactivated.map(h => h.hash) : [] }, 'Blockchain reorganization detected' ) } ) - log.info({ operation: 'subscribe.headers', outcome: 'ok', subscription_id: headerSubscriptionId }, 'Subscribed to header events') - log.info({ operation: 'subscribe.reorgs', outcome: 'ok', subscription_id: reorgSubscriptionId }, 'Subscribed to reorg events') + log.info( + { operation: 'subscribe.headers', outcome: 'ok', subscription_id: headerSubscriptionId }, + 'Subscribed to header events' + ) + log.info( + { operation: 'subscribe.reorgs', outcome: 'ok', subscription_id: reorgSubscriptionId }, + 'Subscribed to reorg events' + ) // Create custom Services instance // This allows configuring which BSV network services to use // Note: Services uses the chain parameter to configure network services - const services = new Services(chain) + const services = createServices(chain, chaintracks) // Create Express app with both v1 and v2 routes const app = express.default() + const resourceProfile = readResourceProfile('CHAINTRACKS') app.disable('x-powered-by') + app.use(initialDoubleSlashCompatibility) app.use(securityHeaders({ environmentPrefix: 'CHAINTRACKS' })) - app.use(corsPolicy({ - environmentPrefix: 'CHAINTRACKS', - methods: ['GET', 'POST', 'OPTIONS'] - })) - app.use(concurrencyLimit('CHAINTRACKS', 200)) + app.use( + corsPolicy({ + environmentPrefix: 'CHAINTRACKS', + methods: ['GET', 'POST', 'OPTIONS'] + }) + ) + app.use( + concurrencyLimit( + 'CHAINTRACKS', + profileValue(resourceProfile, { + small: 32, + standard: 64, + highThroughput: 256 + }) + ) + ) // Body parser for POST requests - app.use(bodyParser.json({ - limit: readBodyLimitBytes('CHAINTRACKS', 256 * 1024) - })) + app.use( + bodyParser.json({ + limit: readBodyLimitBytes('CHAINTRACKS', 256 * 1024) + }) + ) app.use(bodyParserErrorHandler) + app.use( + responseSizeLimit( + 'CHAINTRACKS', + profileValue(resourceProfile, { + small: 1024 * 1024, + standard: 4 * 1024 * 1024, + highThroughput: 32 * 1024 * 1024 + }) + ) + ) + + const healthHandler = (_req: express.Request, res: express.Response) => { + res.setHeader('Cache-Control', 'no-store') + res.status(200).json({ status: 'ok', profile: resourceProfile }) + } + app.get('/healthz', healthHandler) + + const apiRouter = express.Router() + if (routingPrefix !== '') apiRouter.get('/healthz', healthHandler) + + apiRouter.get('/readyz', async (_req: express.Request, res: express.Response) => { + res.setHeader('Cache-Control', 'no-store') + try { + const [listening, height, info] = await Promise.all([ + chaintracks.isListening(), + chaintracks.getPresentHeight(), + chaintracks.getInfo() + ]) + res + .status(listening ? 200 : 503) + .json({ status: listening ? 'ok' : 'starting', height, info }) + } catch (error) { + log.warn({ operation: 'readiness', outcome: 'error', err: error }, 'Chaintracks is not ready') + res.status(503).json({ status: 'error', description: 'Chaintracks is not ready' }) + } + }) // Root endpoint - app.get('/', (_req: express.Request, res: express.Response) => { + apiRouter.get('/', (_req: express.Request, res: express.Response) => { res.json({ status: 'success', value: 'chaintracks-server' }) }) @@ -295,15 +528,19 @@ async function main() { // Mount v1 routes (RPC-style, original API) const v1Routes = createV1Routes({ chaintracks, services, chain }) - app.use('/', v1Routes) + apiRouter.use('/', v1Routes) // Mount v2 routes (RESTful, go-chaintracks compatible) const v2Routes = createV2Routes(chaintracks) - app.use('/v2', v2Routes) + apiRouter.use('/v2', v2Routes) + app.use(routingPrefix || '/', apiRouter) // Start the API server const apiServer = app.listen(port, () => { - log.info({ operation: 'listen', outcome: 'ok', port, chain: `${chain}Net` }, 'API server running') + log.info( + { operation: 'listen', outcome: 'ok', port, chain: `${chain}Net` }, + 'API server running' + ) }) configureHttpServer(apiServer, 'CHAINTRACKS', { requestTimeoutMs: 30_000, @@ -320,24 +557,29 @@ async function main() { const cdnApp = express.default() cdnApp.disable('x-powered-by') cdnApp.use(securityHeaders({ environmentPrefix: 'CHAINTRACKS_CDN' })) - cdnApp.use(corsPolicy({ - environmentPrefix: 'CHAINTRACKS_CDN', - methods: ['GET', 'HEAD', 'OPTIONS'] - })) + cdnApp.use( + corsPolicy({ + environmentPrefix: 'CHAINTRACKS_CDN', + methods: ['GET', 'HEAD', 'OPTIONS'] + }) + ) cdnApp.use(concurrencyLimit('CHAINTRACKS_CDN', 100)) // Serve static files from the bulk headers directory - cdnApp.use('/', express.static(bulkHeadersPath, { - setHeaders: (res: any, filePath: string) => { - // Set appropriate headers for bulk header files - if (filePath.endsWith('.headers')) { - res.setHeader('Content-Type', 'application/octet-stream') - } else if (filePath.endsWith('.json')) { - res.setHeader('Content-Type', 'application/json') + cdnApp.use( + '/', + express.static(bulkHeadersPath, { + setHeaders: (res: any, filePath: string) => { + // Set appropriate headers for bulk header files + if (filePath.endsWith('.headers')) { + res.setHeader('Content-Type', 'application/octet-stream') + } else if (filePath.endsWith('.json')) { + res.setHeader('Content-Type', 'application/json') + } + res.setHeader('Cache-Control', 'public, max-age=3600') } - res.setHeader('Cache-Control', 'public, max-age=3600') - } - })) + }) + ) cdnServer = cdnApp.listen(cdnPort, () => { log.info( @@ -345,7 +587,7 @@ async function main() { operation: 'cdn.listen', outcome: 'ok', cdn_port: cdnPort, - access_url: `http://localhost:${cdnPort}/mainNetBlockHeaders.json`, + access_url: `http://localhost:${cdnPort}/mainNetBlockHeaders.json` }, 'Bulk Headers CDN server running' ) @@ -361,7 +603,10 @@ async function main() { // Perform initial export if CDN is enabled if (enableBulkHeadersCDN) { - log.info({ operation: 'headers.export', context: 'initial' }, 'Performing initial bulk headers export') + log.info( + { operation: 'headers.export', context: 'initial' }, + 'Performing initial bulk headers export' + ) await exportBulkHeaders() } @@ -370,7 +615,10 @@ async function main() { if (enableBulkHeadersCDN) { exportInterval = setInterval(() => { exportBulkHeaders().catch(err => - log.error({ operation: 'headers.export', outcome: 'error', context: 'periodic', err }, 'Periodic export error') + log.error( + { operation: 'headers.export', outcome: 'error', context: 'periodic', err }, + 'Periodic export error' + ) ) }, bulkHeadersAutoExportInterval) } @@ -382,6 +630,7 @@ async function main() { port, cdn_enabled: enableBulkHeadersCDN, cdn_port: enableBulkHeadersCDN ? cdnPort : undefined, + routing_prefix: routingPrefix || '/', v1_endpoints: [ 'GET /getChain', 'GET /getInfo', @@ -391,18 +640,18 @@ async function main() { 'GET /findHeaderHexForHeight?height=N', 'GET /findHeaderHexForBlockHash?hash=X', 'GET /getHeaders?height=N&count=M', - 'POST /addHeaderHex', + 'POST /addHeaderHex' ], v2_endpoints: [ 'GET /v2/network', 'GET /v2/tip', 'GET /v2/header/height/:height', 'GET /v2/header/hash/:hash', - 'GET /v2/headers?height=N&count=M', + 'GET /v2/headers?height=N&count=M' ], cdn_endpoints: enableBulkHeadersCDN ? [`GET /${chain}NetBlockHeaders.json`, 'GET /*.headers'] - : undefined, + : undefined }, 'Chaintracks API server is running' ) @@ -414,13 +663,16 @@ async function main() { // Stop periodic export if running if (exportInterval) { clearInterval(exportInterval) - log.info({ operation: 'shutdown.export_timer', outcome: 'ok' }, 'Stopped periodic export timer') + log.info( + { operation: 'shutdown.export_timer', outcome: 'ok' }, + 'Stopped periodic export timer' + ) } // Stop CDN server if running if (cdnServer) { log.info({ operation: 'shutdown.cdn_server' }, 'Stopping CDN server') - await new Promise((resolve) => { + await new Promise(resolve => { cdnServer.close(() => { log.info({ operation: 'shutdown.cdn_server', outcome: 'ok' }, 'CDN server stopped') resolve() @@ -435,7 +687,7 @@ async function main() { // Stop the API server log.info({ operation: 'shutdown.api_server' }, 'Stopping API server') - await new Promise((resolve) => { + await new Promise(resolve => { apiServer.close(() => { log.info({ operation: 'shutdown.api_server', outcome: 'ok' }, 'API server stopped') resolve() @@ -456,28 +708,40 @@ async function main() { process.on('SIGINT', () => shutdown('SIGINT')) process.on('SIGTERM', () => shutdown('SIGTERM')) - process.on('uncaughtException', (error) => { - log.error({ operation: 'uncaught_exception', outcome: 'error', err: error }, 'Uncaught Exception') + process.on('uncaughtException', error => { + log.error( + { operation: 'uncaught_exception', outcome: 'error', err: error }, + 'Uncaught Exception' + ) shutdown('uncaughtException') }) process.on('unhandledRejection', (reason, promise) => { - log.error({ operation: 'unhandled_rejection', outcome: 'error', err: reason, promise }, 'Unhandled Rejection') + log.error( + { operation: 'unhandled_rejection', outcome: 'error', err: reason, promise }, + 'Unhandled Rejection' + ) shutdown('unhandledRejection') }) } // Wrap startup in a span so a slow/failed boot is visible in traces. -tracer.startActiveSpan('chaintracks.bootstrap', async (span) => { +tracer.startActiveSpan('chaintracks.bootstrap', async span => { const startedAt = Date.now() try { await main() span.setStatus({ code: SpanStatusCode.OK }) - log.info({ operation: 'bootstrap', outcome: 'ok', duration_ms: Date.now() - startedAt }, 'chaintracks-server started') + log.info( + { operation: 'bootstrap', outcome: 'ok', duration_ms: Date.now() - startedAt }, + 'chaintracks-server started' + ) span.end() } catch (error) { span.recordException(error as Error) span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error).message }) - log.error({ operation: 'bootstrap', outcome: 'error', duration_ms: Date.now() - startedAt, err: error }, 'Failed to start server') + log.error( + { operation: 'bootstrap', outcome: 'error', duration_ms: Date.now() - startedAt, err: error }, + 'Failed to start server' + ) span.end() process.exit(1) } diff --git a/infra/chaintracks-server/src/v1-routes.ts b/infra/chaintracks-server/src/v1-routes.ts index eba62176e..3aef81b4d 100644 --- a/infra/chaintracks-server/src/v1-routes.ts +++ b/infra/chaintracks-server/src/v1-routes.ts @@ -7,6 +7,7 @@ import { Router, Request, Response } from 'express' import { Chaintracks, Services } from '@bsv/wallet-toolbox' import { log } from './logger' +import { parseHeaderRange } from './resourceLimits' interface ApiResponse { status: 'success' | 'error' @@ -153,15 +154,7 @@ export function createV1Routes(options: V1RoutesOptions): Router { // GET /getHeaders - Get multiple headers as hex string router.get('/getHeaders', async (req: Request, res: Response) => { try { - const height = Number.parseInt(req.query.height as string, 10) - const count = Number.parseInt(req.query.count as string, 10) - - if (Number.isNaN(height) || height < 0) { - return res.status(400).json(error('ERR_INVALID_PARAMS', 'Invalid or missing height parameter')) - } - if (Number.isNaN(count) || count <= 0) { - return res.status(400).json(error('ERR_INVALID_PARAMS', 'Invalid or missing count parameter')) - } + const { height, count } = parseHeaderRange(req.query as Record) const currentHeight = await chaintracks.currentHeight() if (height < currentHeight - 100) { @@ -190,6 +183,9 @@ export function createV1Routes(options: V1RoutesOptions): Router { res.json(success(hexString)) } catch (err) { + if (err instanceof RangeError) { + return res.status(400).json(error('ERR_INVALID_PARAMS', err.message)) + } log.error({ operation: 'v1.get_headers', outcome: 'error', err }, 'Failed to get headers') res.status(500).json(error('ERR_INTERNAL', 'Failed to get headers')) } diff --git a/infra/chaintracks-server/src/v2-routes.ts b/infra/chaintracks-server/src/v2-routes.ts index 6481c4291..30d16a427 100644 --- a/infra/chaintracks-server/src/v2-routes.ts +++ b/infra/chaintracks-server/src/v2-routes.ts @@ -7,6 +7,7 @@ import { Router, Request, Response } from 'express' import { Chaintracks } from '@bsv/wallet-toolbox' import { log } from './logger' +import { parseHeaderRange } from './resourceLimits' interface ApiResponse { status: 'success' | 'error' @@ -37,11 +38,18 @@ function reverseHex(hex: string): Buffer { // Convert header to 80-byte binary format // Note: previousHash and merkleRoot are byte-reversed in JSON (display format) // but need to be in internal byte order for binary serialization -function headerToBytes(header: { version: number; previousHash: string; merkleRoot: string; time: number; bits: number; nonce: number }): Buffer { +function headerToBytes(header: { + version: number + previousHash: string + merkleRoot: string + time: number + bits: number + nonce: number +}): Buffer { const buf = Buffer.alloc(80) buf.writeUInt32LE(header.version, 0) - reverseHex(header.previousHash).copy(buf, 4) // Reverse from display to internal - reverseHex(header.merkleRoot).copy(buf, 36) // Reverse from display to internal + reverseHex(header.previousHash).copy(buf, 4) // Reverse from display to internal + reverseHex(header.merkleRoot).copy(buf, 36) // Reverse from display to internal buf.writeUInt32LE(header.time, 68) buf.writeUInt32LE(header.bits, 72) buf.writeUInt32LE(header.nonce, 76) @@ -62,6 +70,17 @@ export function createV2Routes(chaintracks: Chaintracks): Router { } }) + // GET /v2/height - Get current chain height (go-chaintracks compatible) + router.get('/height', async (_req: Request, res: Response) => { + try { + res.set('Cache-Control', 'public, max-age=60') + res.json(success({ height: await chaintracks.getPresentHeight() })) + } catch (err) { + log.error({ operation: 'v2.get_height', outcome: 'error', err }, 'Failed to get chain height') + res.status(500).json(error('ERR_INTERNAL', 'Failed to get chain height')) + } + }) + // GET /v2/tip - Get chain tip header router.get('/tip', async (_req: Request, res: Response) => { try { @@ -77,6 +96,110 @@ export function createV2Routes(chaintracks: Chaintracks): Router { } }) + // GET /v2/tip/stream - SSE stream compatible with go-chaintracks/Arcade. + router.get('/tip/stream', async (req: Request, res: Response) => { + res.status(200) + res.set({ + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }) + res.flushHeaders() + + let subscriptionId: string | undefined + let closed = false + const keepalive = setInterval(() => { + if (!res.writableEnded) res.write(': keepalive\n\n') + }, 15000) + const cleanup = () => { + closed = true + clearInterval(keepalive) + if (subscriptionId != null) { + const id = subscriptionId + subscriptionId = undefined + chaintracks.unsubscribe(id).catch(err => { + log.warn( + { operation: 'v2.tip_stream.unsubscribe', outcome: 'error', err }, + 'Failed to unsubscribe tip stream' + ) + }) + } + } + req.once('close', cleanup) + + try { + subscriptionId = await chaintracks.subscribeHeaders(header => { + if (!res.writableEnded) res.write(`data: ${JSON.stringify(header)}\n\n`) + }) + if (closed) { + const id = subscriptionId + subscriptionId = undefined + await chaintracks.unsubscribe(id) + return + } + const tip = await chaintracks.findChainTipHeader() + if (!res.writableEnded) res.write(`data: ${JSON.stringify(tip)}\n\n`) + } catch (err) { + log.error({ operation: 'v2.tip_stream', outcome: 'error', err }, 'Failed to start tip stream') + cleanup() + if (!res.writableEnded) res.end() + } + }) + + // GET /v2/reorg/stream - SSE reorganization stream. + router.get('/reorg/stream', async (req: Request, res: Response) => { + res.status(200) + res.set({ + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }) + res.flushHeaders() + + let subscriptionId: string | undefined + let closed = false + const keepalive = setInterval(() => { + if (!res.writableEnded) res.write(': keepalive\n\n') + }, 15000) + const cleanup = () => { + closed = true + clearInterval(keepalive) + if (subscriptionId != null) { + const id = subscriptionId + subscriptionId = undefined + chaintracks.unsubscribe(id).catch(err => { + log.warn( + { operation: 'v2.reorg_stream.unsubscribe', outcome: 'error', err }, + 'Failed to unsubscribe reorg stream' + ) + }) + } + } + req.once('close', cleanup) + + try { + subscriptionId = await chaintracks.subscribeReorgs( + (depth, oldTip, newTip, deactivatedHeaders) => { + if (!res.writableEnded) { + res.write(`data: ${JSON.stringify({ depth, oldTip, newTip, deactivatedHeaders })}\n\n`) + } + } + ) + if (closed) { + const id = subscriptionId + subscriptionId = undefined + await chaintracks.unsubscribe(id) + } + } catch (err) { + log.error( + { operation: 'v2.reorg_stream', outcome: 'error', err }, + 'Failed to start reorg stream' + ) + cleanup() + if (!res.writableEnded) res.end() + } + }) + // GET /v2/header/height/:height - Get header by height router.get('/header/height/:height', async (req: Request, res: Response) => { try { @@ -98,7 +221,10 @@ export function createV2Routes(chaintracks: Chaintracks): Router { } res.json(success(header)) } catch (err) { - log.error({ operation: 'v2.get_header_by_height', outcome: 'error', err }, 'Failed to get header') + log.error( + { operation: 'v2.get_header_by_height', outcome: 'error', err }, + 'Failed to get header' + ) res.status(500).json(error('ERR_INTERNAL', 'Failed to get header')) } }) @@ -125,7 +251,10 @@ export function createV2Routes(chaintracks: Chaintracks): Router { res.json(success(header)) } catch (err) { - log.error({ operation: 'v2.get_header_by_hash', outcome: 'error', err }, 'Failed to get header') + log.error( + { operation: 'v2.get_header_by_hash', outcome: 'error', err }, + 'Failed to get header' + ) res.status(500).json(error('ERR_INTERNAL', 'Failed to get header')) } }) @@ -133,15 +262,7 @@ export function createV2Routes(chaintracks: Chaintracks): Router { // GET /v2/headers?height=N&count=M - Get multiple headers as binary router.get('/headers', async (req: Request, res: Response) => { try { - const height = Number.parseInt(req.query.height as string, 10) - const count = Number.parseInt(req.query.count as string, 10) - - if (Number.isNaN(height) || height < 0) { - return res.status(400).json(error('ERR_INVALID_PARAMS', 'Invalid or missing height parameter')) - } - if (Number.isNaN(count) || count <= 0) { - return res.status(400).json(error('ERR_INVALID_PARAMS', 'Invalid or missing count parameter')) - } + const { height, count } = parseHeaderRange(req.query as Record) const currentHeight = await chaintracks.currentHeight() if (height < currentHeight - 100) { @@ -161,6 +282,9 @@ export function createV2Routes(chaintracks: Chaintracks): Router { res.set('Content-Type', 'application/octet-stream') res.send(Buffer.concat(buffers)) } catch (err) { + if (err instanceof RangeError) { + return res.status(400).json(error('ERR_INVALID_PARAMS', err.message)) + } log.error({ operation: 'v2.get_headers', outcome: 'error', err }, 'Failed to get headers') res.status(500).json(error('ERR_INTERNAL', 'Failed to get headers')) } @@ -209,7 +333,10 @@ export function createV2Routes(chaintracks: Chaintracks): Router { res.set('X-Block-Height', String(header.height)) res.send(headerToBytes(header)) } catch (err) { - log.error({ operation: 'v2.get_header_by_height_bin', outcome: 'error', err }, 'Failed to get header') + log.error( + { operation: 'v2.get_header_by_height_bin', outcome: 'error', err }, + 'Failed to get header' + ) res.status(500).json(error('ERR_INTERNAL', 'Failed to get header')) } }) @@ -238,7 +365,10 @@ export function createV2Routes(chaintracks: Chaintracks): Router { res.set('X-Block-Height', String(header.height)) res.send(headerToBytes(header)) } catch (err) { - log.error({ operation: 'v2.get_header_by_hash_bin', outcome: 'error', err }, 'Failed to get header') + log.error( + { operation: 'v2.get_header_by_hash_bin', outcome: 'error', err }, + 'Failed to get header' + ) res.status(500).json(error('ERR_INTERNAL', 'Failed to get header')) } }) @@ -246,15 +376,7 @@ export function createV2Routes(chaintracks: Chaintracks): Router { // GET /v2/headers.bin?height=N&count=M - Get multiple headers as binary (80 bytes each) router.get('/headers.bin', async (req: Request, res: Response) => { try { - const height = Number.parseInt(req.query.height as string, 10) - const count = Number.parseInt(req.query.count as string, 10) - - if (Number.isNaN(height) || height < 0) { - return res.status(400).json(error('ERR_INVALID_PARAMS', 'Invalid or missing height parameter')) - } - if (Number.isNaN(count) || count <= 0) { - return res.status(400).json(error('ERR_INVALID_PARAMS', 'Invalid or missing count parameter')) - } + const { height, count } = parseHeaderRange(req.query as Record) const currentHeight = await chaintracks.currentHeight() if (height < currentHeight - 100) { @@ -278,6 +400,9 @@ export function createV2Routes(chaintracks: Chaintracks): Router { res.set('X-Header-Count', String(headerCount)) res.send(Buffer.concat(buffers)) } catch (err) { + if (err instanceof RangeError) { + return res.status(400).json(error('ERR_INVALID_PARAMS', err.message)) + } log.error({ operation: 'v2.get_headers_bin', outcome: 'error', err }, 'Failed to get headers') res.status(500).json(error('ERR_INTERNAL', 'Failed to get headers')) } diff --git a/infra/message-box-server/.env.example b/infra/message-box-server/.env.example index dfeed1360..187a2d5b8 100644 --- a/infra/message-box-server/.env.example +++ b/infra/message-box-server/.env.example @@ -30,8 +30,11 @@ LOG_LEVEL=info # MESSAGE_BOX_STRICT_TRANSPORT_SECURITY=false # Bounded HTTP/WebSocket resource policy. +MESSAGE_BOX_RESOURCE_PROFILE=standard MESSAGE_BOX_MAX_BODY_BYTES=4194304 -MESSAGE_BOX_MAX_CONCURRENT_REQUESTS=200 +MESSAGE_BOX_MAX_RESPONSE_BYTES=8388608 +MESSAGE_BOX_MAX_CONCURRENT_REQUESTS=24 +MESSAGE_BOX_MAX_CONNECTIONS=1000 MESSAGE_BOX_WEBSOCKET_MAX_BODY_BYTES=1048576 MESSAGE_BOX_REQUEST_TIMEOUT_MS=60000 MESSAGE_BOX_HEADERS_TIMEOUT_MS=15000 @@ -39,6 +42,42 @@ MESSAGE_BOX_KEEP_ALIVE_TIMEOUT_MS=5000 MESSAGE_BOX_SOCKET_TIMEOUT_MS=60000 MESSAGE_BOX_MAX_REQUESTS_PER_SOCKET=1000 +# Message, page, retained-state, maintenance, and database policy. Every +# resource ceiling accepts -1/unlimited as an explicit operator opt-out. +MESSAGE_BOX_MAX_MESSAGE_BODY_BYTES=1048576 +MESSAGE_BOX_MAX_RECIPIENTS=100 +MESSAGE_BOX_LIST_DEFAULT_LIMIT=1000 +MESSAGE_BOX_LIST_MAX_LIMIT=1000 +MESSAGE_BOX_LIST_MAX_OFFSET=100000 +MESSAGE_BOX_LIST_MAX_RESPONSE_BYTES=8388608 +MESSAGE_BOX_MAX_INBOX_MESSAGES=10000 +MESSAGE_BOX_MAX_INBOX_BYTES=1073741824 +MESSAGE_BOX_MAX_SENDER_MESSAGES=10000 +MESSAGE_BOX_MAX_SENDER_BYTES=1073741824 +MESSAGE_BOX_MAX_ACKNOWLEDGMENT_IDS=1000 +MESSAGE_BOX_MAX_NOTIFICATION_DEVICES=100 +MESSAGE_BOX_NOTIFICATION_RECIPIENT_CONCURRENCY=4 +MESSAGE_BOX_FCM_SEND_CONCURRENCY=10 +MESSAGE_BOX_WEBSOCKET_MAX_CONCURRENT_SENDS=4 +MESSAGE_BOX_WEBSOCKET_SEND_RATE_LIMIT=300 +MESSAGE_BOX_WEBSOCKET_MAX_RECIPIENT_CONNECTIONS=25 +MESSAGE_BOX_RETENTION_DAYS=30 +MESSAGE_BOX_RETENTION_CLEANUP_INTERVAL_MS=900000 +MESSAGE_BOX_RETENTION_CLEANUP_BATCH_SIZE=1000 +MESSAGE_BOX_DB_POOL_MIN=0 +MESSAGE_BOX_DB_POOL_MAX=7 +MESSAGE_BOX_DB_IDLE_TIMEOUT_MS=15000 + +# Optional BRC-105 operator monetization (disabled for compatibility). +MESSAGE_BOX_MONETIZATION_ENABLED=false +MESSAGE_BOX_PRICE_BASE_SATOSHIS=50 +MESSAGE_BOX_PRICE_PER_RECIPIENT_SATOSHIS=5 +MESSAGE_BOX_PRICE_PER_KIB_SATOSHIS=5 +MESSAGE_BOX_PRICE_STORAGE_MIB_MONTH_SATOSHIS=1000 +MESSAGE_BOX_PRICE_LIST_PAGE_SATOSHIS=5 +MESSAGE_BOX_PRICE_UNLIMITED_RETENTION_MONTHS=12 +# MESSAGE_BOX_ROUTE_PRICES_JSON={"/healthz":0,"/listMessages":50} + # Firebase push notifications (optional — defaults to disabled) ENABLE_FIREBASE=false # FIREBASE_PROJECT_ID=your-project-id diff --git a/infra/message-box-server/README.md b/infra/message-box-server/README.md index 6fa125f9e..269ad239f 100644 --- a/infra/message-box-server/README.md +++ b/infra/message-box-server/README.md @@ -9,6 +9,9 @@ The maintained source lives in [`bsv-blockchain/ts-stack`](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/message-box-server). The service is distributed as a container; it is not a public npm package. +See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) +for all runtime ceilings, BRC-105 pricing, capacity evidence, and HPA prerequisites. + ## Runtime model - Node.js 24 @@ -97,6 +100,11 @@ Important optional configuration: | `MESSAGE_BOX_CORS_ALLOWED_ORIGINS` | Exact origins for allowlist mode | | `MESSAGE_BOX_MAX_BODY_BYTES` | 4 MiB | | `MESSAGE_BOX_WEBSOCKET_MAX_BODY_BYTES` | 1 MiB | +| `MESSAGE_BOX_WEBSOCKET_MAX_CONCURRENT_SENDS` | 4 | +| `MESSAGE_BOX_WEBSOCKET_SEND_RATE_LIMIT` | 300 authenticated sends/minute/socket | +| `MESSAGE_BOX_WEBSOCKET_MAX_RECIPIENT_CONNECTIONS` | 25 notification targets/message | +| `MESSAGE_BOX_NOTIFICATION_RECIPIENT_CONCURRENCY` | 4 recipient notification workers | +| `MESSAGE_BOX_FCM_SEND_CONCURRENCY` | 10 device-send workers/recipient | | `MESSAGE_BOX_PRE_AUTH_RATE_LIMIT_MAX` | 300 per minute per IP | | `MESSAGE_BOX_AUTHENTICATED_RATE_LIMIT_MAX` | 1,000 per minute per identity | diff --git a/infra/message-box-server/knexfile.ts b/infra/message-box-server/knexfile.ts index 12af123b2..155115f47 100644 --- a/infra/message-box-server/knexfile.ts +++ b/infra/message-box-server/knexfile.ts @@ -7,6 +7,27 @@ const connectionConfig = ? JSON.parse(process.env.KNEX_DB_CONNECTION) : undefined +function readPoolValue(name: string, fallback: number, allowZero = false): number { + const raw = process.env[name] + if (raw == null || raw.trim() === '') return fallback + const pattern = allowZero ? /^\d+$/ : /^[1-9]\d*$/ + if (!pattern.test(raw)) { + throw new Error(`${name} must be ${allowZero ? 'a non-negative' : 'a positive'} integer`) + } + const value = Number(raw) + if (!Number.isSafeInteger(value)) throw new Error(`${name} must be a safe integer`) + return value +} + +const pool = { + min: readPoolValue('MESSAGE_BOX_DB_POOL_MIN', 0, true), + max: readPoolValue('MESSAGE_BOX_DB_POOL_MAX', 7), + idleTimeoutMillis: readPoolValue('MESSAGE_BOX_DB_IDLE_TIMEOUT_MS', 15_000) +} +if (pool.min > pool.max) { + throw new Error('MESSAGE_BOX_DB_POOL_MIN must not exceed MESSAGE_BOX_DB_POOL_MAX') +} + const config: Knex.Config = { client: process.env.KNEX_DB_CLIENT ?? 'mysql2', connection: connectionConfig, @@ -14,11 +35,7 @@ const config: Knex.Config = { migrations: { directory: './out/src/migrations' }, - pool: { - min: 0, - max: 7, - idleTimeoutMillis: 15000 - } + pool } const knexfile: { [key: string]: Knex.Config } = { diff --git a/infra/message-box-server/src/app.ts b/infra/message-box-server/src/app.ts index d93138c79..afdbfa112 100644 --- a/infra/message-box-server/src/app.ts +++ b/infra/message-box-server/src/app.ts @@ -22,7 +22,7 @@ import * as dotenv from 'dotenv' import express, { Express } from 'express' import bodyParser from 'body-parser' import { Logger } from './utils/logger.js' -import { Setup } from '@bsv/wallet-toolbox' +import { KnexSessionManager, Setup } from '@bsv/wallet-toolbox' import knexLib, { Knex } from 'knex' import knexConfig from '../knexfile.js' import type { WalletInterface } from '@bsv/sdk' @@ -37,9 +37,17 @@ import { bodyParserErrorHandler, concurrencyLimit, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders } from './security/edgePolicy.js' +import { calculateConfiguredRequestPrice } from './config/pricing.js' +import { readMessageBoxResourceConfig } from './config/resources.js' +import { KnexPaymentReplayStore } from './security/KnexPaymentReplayStore.js' ;(global.self as any) = { crypto } dotenv.config() @@ -78,6 +86,20 @@ export const knex: Knex = : knexConfig.development ) +const authSessionTtlMs = readResourceLimit( + 'MESSAGE_BOX', + 'AUTH_SESSION_TTL_MS', + 24 * 60 * 60 * 1_000 +) +if (authSessionTtlMs === -1) { + throw new Error('MESSAGE_BOX_AUTH_SESSION_TTL_MS must be finite') +} +export const sessionManager = new KnexSessionManager(knex, { ttlMs: authSessionTtlMs }) +export const paymentReplayStore = new KnexPaymentReplayStore( + knex, + readResourceLimit('MESSAGE_BOX', 'PAYMENT_REPLAY_TTL_DAYS', 365) +) + // Wallet initialization logic let _wallet: WalletInterface | undefined let _resolveReady: () => void @@ -145,6 +167,8 @@ export const appReady = (async () => { * @throws If wallet is not available when needed */ export async function useRoutes(): Promise { + const profile = readResourceProfile('MESSAGE_BOX') + app.use(initialDoubleSlashCompatibility) app.use( securityHeaders({ environmentPrefix: 'MESSAGE_BOX', @@ -158,17 +182,30 @@ export async function useRoutes(): Promise { methods: ['GET', 'POST', 'OPTIONS'] }) ) - app.use(concurrencyLimit('MESSAGE_BOX', 200)) + app.use( + concurrencyLimit( + 'MESSAGE_BOX', + profileValue(profile, { small: 8, standard: 24, highThroughput: 96 }) + ) + ) app.use( rateLimit(rateLimitOptions('MESSAGE_BOX_PRE_AUTH_RATE_LIMIT', { windowMs: 60_000, limit: 300 })) ) app.use( bodyParser.json({ - limit: readBodyLimitBytes('MESSAGE_BOX', 4 * 1024 * 1024), + limit: readBodyLimitBytes( + 'MESSAGE_BOX', + profileValue(profile, { + small: 1024 * 1024, + standard: 4 * 1024 * 1024, + highThroughput: 16 * 1024 * 1024 + }) + ), type: 'application/json' }) ) app.use(bodyParserErrorHandler) + app.use(responseSizeLimit('MESSAGE_BOX', readMessageBoxResourceConfig().listMaxResponseBytes)) // Enable Swagger docs setupSwagger(app) @@ -183,17 +220,21 @@ export async function useRoutes(): Promise { app.use( createAuthMiddleware({ wallet: _wallet, + sessionManager, logger: console }) ) + // Auth middleware intercepts responses for BRC-104 signing. Install the + // limiter again after that interception so the signed materialization is + // bounded even with older compatible auth-middleware releases. + app.use(responseSizeLimit('MESSAGE_BOX', readMessageBoxResourceConfig().listMaxResponseBytes)) registerMessageBoxPostAuthRoutes( app, { wallet: _wallet, - // Message delivery is free unless an embedding operator injects a price - // calculator through the composable context. - calculateRequestPrice: () => 0 + calculateRequestPrice: calculateConfiguredRequestPrice, + paymentReplayStore }, ROUTING_PREFIX ) diff --git a/infra/message-box-server/src/compose.lifecycle.test.ts b/infra/message-box-server/src/compose.lifecycle.test.ts index bc9841c97..5696ca42c 100644 --- a/infra/message-box-server/src/compose.lifecycle.test.ts +++ b/infra/message-box-server/src/compose.lifecycle.test.ts @@ -1,7 +1,4 @@ -import { - closeMessageBoxWebSockets, - disconnectAuthenticatedSockets -} from './compose.js' +import { closeMessageBoxWebSockets, disconnectAuthenticatedSockets } from './compose.js' describe('Message Box WebSocket lifecycle', () => { it('uses the package-owned close lifecycle when it is available', async () => { diff --git a/infra/message-box-server/src/compose.ts b/infra/message-box-server/src/compose.ts index 3420c54fe..a4bb55956 100644 --- a/infra/message-box-server/src/compose.ts +++ b/infra/message-box-server/src/compose.ts @@ -21,7 +21,13 @@ import { Logger } from './utils/logger.js' import { bindMessageBoxRuntime } from './runtimeDeps.js' import type { MessageBoxContext } from './context.js' import { authenticatedIdentityKey, rateLimitOptions } from './security/rateLimitPolicy.js' -import { readCorsOriginSetting, readBodyLimitBytes } from './security/edgePolicy.js' +import { + readCorsOriginSetting, + readBodyLimitBytes, + responseSizeLimit +} from './security/edgePolicy.js' +import { readMessageBoxResourceConfig } from './config/resources.js' +import { readMessageBoxPricingConfig } from './config/pricing.js' import { authenticatedWebSocketIdentity, isIdentityOwnedRoom, @@ -52,9 +58,32 @@ type ClosableAuthSocketServer = AuthSocketServer & { type DisconnectableAuthSocket = Pick -export function disconnectAuthenticatedSockets( - sockets: Iterable -): void { +function validateWebSocketMessage( + roomId: unknown, + message: unknown +): { reason: string; logValue?: unknown } | null { + if (typeof roomId !== 'string' || roomId.trim() === '') { + return { reason: 'Invalid room ID', logValue: roomId } + } + if (typeof message !== 'object' || message == null) { + return { reason: 'Invalid message object', logValue: message } + } + const candidate = message as { body?: unknown; recipient?: unknown } + if (typeof candidate.body !== 'string' || candidate.body.trim() === '') { + return { reason: 'Invalid message body' } + } + if (typeof candidate.recipient !== 'string') { + return { reason: 'Invalid recipient identity key' } + } + try { + PublicKey.fromString(candidate.recipient) + } catch { + return { reason: 'Invalid recipient identity key' } + } + return null +} + +export function disconnectAuthenticatedSockets(sockets: Iterable): void { for (const socket of sockets) { socket.ioSocket.disconnect(true) } @@ -102,10 +131,12 @@ export function registerMessageBoxPreAuthRoutes( /** Payment middleware (after auth) + postAuth route handlers. */ export function registerMessageBoxPostAuthRoutes( router: MessageBoxRouter, - ctx: Pick, + ctx: Pick, routingPrefix: string = '', authenticatedRateLimitOptions: Partial = {} ): void { + const resources = readMessageBoxResourceConfig() + router.use(responseSizeLimit('MESSAGE_BOX', resources.listMaxResponseBytes)) router.use( rateLimit( rateLimitOptions( @@ -123,7 +154,8 @@ export function registerMessageBoxPostAuthRoutes( createPaymentMiddleware({ wallet: ctx.wallet, calculateRequestPrice: async req => - await Promise.resolve(ctx.calculateRequestPrice(req as unknown as ExpressRequest)) + await Promise.resolve(ctx.calculateRequestPrice(req as unknown as ExpressRequest)), + replayStore: ctx.paymentReplayStore }) ) @@ -159,6 +191,7 @@ export function attachMessageBoxWebSockets( const io = new AuthSocketServer(httpServer, { wallet: ctx.wallet, + sessionManager: ctx.sessionManager, maxHttpBufferSize: readBodyLimitBytes('MESSAGE_BOX_WEBSOCKET', 1024 * 1024), cors: { origin: readCorsOriginSetting('MESSAGE_BOX'), @@ -169,9 +202,14 @@ export function attachMessageBoxWebSockets( // Map to store authenticated identity keys const authenticatedSockets = new Map() const connectedSockets = new Map() + const resources = readMessageBoxResourceConfig() + const pricing = readMessageBoxPricingConfig() webSocketState.set(io, { authenticatedSockets, connectedSockets }) io.on('connection', socket => { + let activeSendEvents = 0 + let sendRateWindowStartedAt = Date.now() + let sendEventsInWindow = 0 connectedSockets.set(socket.id, socket) Logger.log('[WEBSOCKET] New connection established.') @@ -248,31 +286,43 @@ export function attachMessageBoxWebSockets( return } - Logger.log(`[WEBSOCKET] Processing sendMessage for room: ${roomId}`) - - try { - if (typeof roomId !== 'string' || roomId.trim() === '') { - Logger.error('[WEBSOCKET ERROR] Invalid roomId:', roomId) - await socket.emit('messageFailed', { reason: 'Invalid room ID' }) - return - } - - if (typeof message !== 'object' || message == null) { - Logger.error('[WEBSOCKET ERROR] Invalid message object:', message) - await socket.emit('messageFailed', { reason: 'Invalid message object' }) - return - } + if ( + resources.webSocketMaxConcurrentSends !== -1 && + activeSendEvents >= resources.webSocketMaxConcurrentSends + ) { + await socket.emit('messageFailed', { + reason: 'Too many concurrent WebSocket sends', + code: 'ERR_WEBSOCKET_CONCURRENCY_LIMIT' + }) + return + } - if (typeof message.body !== 'string' || message.body.trim() === '') { - Logger.error('[WEBSOCKET ERROR] Invalid message body.') - await socket.emit('messageFailed', { reason: 'Invalid message body' }) - return - } + const now = Date.now() + if (now - sendRateWindowStartedAt >= 60_000) { + sendRateWindowStartedAt = now + sendEventsInWindow = 0 + } + if ( + resources.webSocketSendRateLimit !== -1 && + sendEventsInWindow >= resources.webSocketSendRateLimit + ) { + await socket.emit('messageFailed', { + reason: 'WebSocket send rate limit exceeded', + code: 'ERR_WEBSOCKET_RATE_LIMITED' + }) + return + } + sendEventsInWindow += 1 + activeSendEvents += 1 - try { - PublicKey.fromString(message.recipient) - } catch { - await socket.emit('messageFailed', { reason: 'Invalid recipient identity key' }) + try { + const validationFailure = validateWebSocketMessage(roomId, message) + if (validationFailure != null) { + Logger.error('[WEBSOCKET ERROR] Rejected invalid sendMessage event:', { + reason: validationFailure.reason, + value: validationFailure.logValue + }) + await socket.emit('messageFailed', { reason: validationFailure.reason }) return } @@ -284,10 +334,23 @@ export function attachMessageBoxWebSockets( return } + Logger.log(`[WEBSOCKET] Processing sendMessage for room: ${roomId}`) + + // BRC-105 payments are authenticated HTTP exchanges. Refuse the + // legacy write event when monetization is enabled so current clients + // immediately exercise their existing AuthFetch fallback instead of + // bypassing the payment middleware. + if (pricing.enabled) { + await socket.emit(`sendMessageAck-${roomId}`, { + status: 'error', + code: 'ERR_PAYMENT_REQUIRES_AUTHFETCH' + }) + return + } + // Reuse the HTTP route's complete validation, recipient-permission, - // fee, payment, duplicate, and persistence policy. WebSocket sends - // that require payment return an error so the client can use its - // existing authenticated HTTP fallback. + // duplicate, quota, and persistence policy. Paid deployments are + // routed through AuthFetch above rather than this legacy event. let routeStatus = 200 let routeBody: any const routeResponse = { @@ -328,7 +391,15 @@ export function attachMessageBoxWebSockets( messageId: message.messageId }) - const recipientSockets = recipientSocketIds(authenticatedSockets, message.recipient) + const recipientSocketIdsForMessage = recipientSocketIds( + authenticatedSockets, + message.recipient + ) + const boundedRecipientSocketIds = + resources.webSocketMaxRecipientConnections === -1 + ? recipientSocketIdsForMessage + : recipientSocketIdsForMessage.slice(0, resources.webSocketMaxRecipientConnections) + const recipientSockets = boundedRecipientSocketIds .map(socketId => connectedSockets.get(socketId)) .filter(recipientSocket => recipientSocket != null) await Promise.all( @@ -347,6 +418,8 @@ export function attachMessageBoxWebSockets( } catch (error) { Logger.error('[WEBSOCKET ERROR] Unexpected failure in sendMessage handler:', error) await socket.emit('messageFailed', { reason: 'Unexpected error occurred' }) + } finally { + activeSendEvents -= 1 } } ) diff --git a/infra/message-box-server/src/config/pricing.ts b/infra/message-box-server/src/config/pricing.ts new file mode 100644 index 000000000..2ddcbb4b4 --- /dev/null +++ b/infra/message-box-server/src/config/pricing.ts @@ -0,0 +1,113 @@ +import type { Request } from 'express' +import { readMessageBoxResourceConfig } from './resources.js' + +export interface MessageBoxPricingConfig { + enabled: boolean + baseSatoshis: number + perRecipientSatoshis: number + perKiBSatoshis: number + storageMiBMonthSatoshis: number + unlimitedRetentionMonths: number + listPageSatoshis: number + routePrices: Readonly> +} + +function readBoolean(name: string, fallback: boolean): boolean { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + if (value === 'true') return true + if (value === 'false') return false + throw new Error(`${name} must be true or false`) +} + +function readSatoshis(name: string, fallback: number): number { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + if (!/^\d+$/.test(value)) throw new Error(`${name} must be a non-negative integer`) + const parsed = Number(value) + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} must be a safe integer`) + return parsed +} + +function readRoutePrices(): Readonly> { + const raw = process.env.MESSAGE_BOX_ROUTE_PRICES_JSON + if (raw == null || raw.trim() === '') return {} + const parsed: unknown = JSON.parse(raw) + if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('MESSAGE_BOX_ROUTE_PRICES_JSON must be a JSON object') + } + const prices: Record = {} + for (const [route, value] of Object.entries(parsed)) { + if ( + !route.startsWith('/') || + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < 0 + ) { + throw new Error( + 'MESSAGE_BOX_ROUTE_PRICES_JSON must map absolute paths to non-negative integers' + ) + } + prices[route] = value + } + return prices +} + +export function readMessageBoxPricingConfig(): MessageBoxPricingConfig { + return { + enabled: readBoolean('MESSAGE_BOX_MONETIZATION_ENABLED', false), + baseSatoshis: readSatoshis('MESSAGE_BOX_PRICE_BASE_SATOSHIS', 50), + perRecipientSatoshis: readSatoshis('MESSAGE_BOX_PRICE_PER_RECIPIENT_SATOSHIS', 5), + perKiBSatoshis: readSatoshis('MESSAGE_BOX_PRICE_PER_KIB_SATOSHIS', 5), + storageMiBMonthSatoshis: readSatoshis('MESSAGE_BOX_PRICE_STORAGE_MIB_MONTH_SATOSHIS', 1_000), + unlimitedRetentionMonths: readSatoshis('MESSAGE_BOX_PRICE_UNLIMITED_RETENTION_MONTHS', 12), + listPageSatoshis: readSatoshis('MESSAGE_BOX_PRICE_LIST_PAGE_SATOSHIS', 5), + routePrices: readRoutePrices() + } +} + +function requestPath(req: Request): string { + const path = req.path || req.url.split('?')[0] + return path.startsWith('/') ? path : `/${path}` +} + +export function calculateConfiguredRequestPrice(req: Request): number { + const pricing = readMessageBoxPricingConfig() + if (!pricing.enabled) return 0 + + const path = requestPath(req) + const routePrice = pricing.routePrices[path] + if (routePrice != null) return routePrice + + if (path.endsWith('/sendMessage')) { + const message = req.body?.message + const recipientsRaw = message?.recipients ?? message?.recipient + const recipientCount = Array.isArray(recipientsRaw) ? recipientsRaw.length : 1 + const body = typeof message?.body === 'string' ? message.body : '' + const bodyBytes = Buffer.byteLength(body, 'utf8') + const resource = readMessageBoxResourceConfig() + const retentionMonths = + resource.retentionDays === -1 ? pricing.unlimitedRetentionMonths : resource.retentionDays / 30 + const storageSatoshis = + bodyBytes === 0 + ? 0 + : Math.ceil((bodyBytes / (1024 * 1024)) * retentionMonths * pricing.storageMiBMonthSatoshis) + return ( + pricing.baseSatoshis + + Math.max(1, recipientCount) * pricing.perRecipientSatoshis + + Math.ceil(bodyBytes / 1024) * pricing.perKiBSatoshis + + storageSatoshis + ) + } + + if (path.endsWith('/listMessages')) { + const requested = Number(req.body?.limit ?? readMessageBoxResourceConfig().listDefaultLimit) + const pages = + Number.isSafeInteger(requested) && requested > 0 + ? Math.max(1, Math.ceil(requested / 1_000)) + : 1 + return pricing.baseSatoshis + pages * pricing.listPageSatoshis + } + + return pricing.baseSatoshis +} diff --git a/infra/message-box-server/src/config/resourceSafety.test.ts b/infra/message-box-server/src/config/resourceSafety.test.ts new file mode 100644 index 000000000..3ff9376fc --- /dev/null +++ b/infra/message-box-server/src/config/resourceSafety.test.ts @@ -0,0 +1,125 @@ +import type { Request } from 'express' +import { calculateConfiguredRequestPrice, readMessageBoxPricingConfig } from './pricing.js' +import { readMessageBoxResourceConfig } from './resources.js' + +const RESOURCE_ENV = [ + 'MESSAGE_BOX_RESOURCE_PROFILE', + 'MESSAGE_BOX_LIST_DEFAULT_LIMIT', + 'MESSAGE_BOX_LIST_MAX_LIMIT', + 'MESSAGE_LIST_BATCH_SIZE', + 'MESSAGE_BOX_MAX_INBOX_MESSAGES', + 'MESSAGE_BOX_NOTIFICATION_RECIPIENT_CONCURRENCY', + 'MESSAGE_BOX_FCM_SEND_CONCURRENCY', + 'MESSAGE_BOX_WEBSOCKET_MAX_CONCURRENT_SENDS', + 'MESSAGE_BOX_WEBSOCKET_SEND_RATE_LIMIT', + 'MESSAGE_BOX_WEBSOCKET_MAX_RECIPIENT_CONNECTIONS', + 'MESSAGE_BOX_RETENTION_DAYS', + 'MESSAGE_BOX_MONETIZATION_ENABLED', + 'MESSAGE_BOX_PRICE_BASE_SATOSHIS', + 'MESSAGE_BOX_PRICE_PER_RECIPIENT_SATOSHIS', + 'MESSAGE_BOX_PRICE_PER_KIB_SATOSHIS', + 'MESSAGE_BOX_PRICE_STORAGE_MIB_MONTH_SATOSHIS', + 'MESSAGE_BOX_PRICE_UNLIMITED_RETENTION_MONTHS', + 'MESSAGE_BOX_PRICE_LIST_PAGE_SATOSHIS', + 'MESSAGE_BOX_ROUTE_PRICES_JSON' +] as const + +describe('Message Box resource safety configuration', () => { + afterEach(() => { + for (const name of RESOURCE_ENV) delete process.env[name] + }) + + it('uses a bounded 1000-message default and accepts explicit unlimited overrides', () => { + expect(readMessageBoxResourceConfig()).toEqual( + expect.objectContaining({ listDefaultLimit: 1_000, listMaxLimit: 1_000 }) + ) + + process.env.MESSAGE_BOX_LIST_DEFAULT_LIMIT = '-1' + process.env.MESSAGE_BOX_LIST_MAX_LIMIT = 'unlimited' + process.env.MESSAGE_BOX_MAX_INBOX_MESSAGES = '-1' + expect(readMessageBoxResourceConfig()).toEqual( + expect.objectContaining({ + listDefaultLimit: -1, + listMaxLimit: -1, + maxInboxMessages: -1 + }) + ) + }) + + it('fails fast when a default exceeds the configured maximum', () => { + process.env.MESSAGE_BOX_LIST_DEFAULT_LIMIT = '1001' + expect(() => readMessageBoxResourceConfig()).toThrow('must not exceed') + }) + + it('bounds WebSocket writes and notification fan-out by default', () => { + expect(readMessageBoxResourceConfig()).toEqual( + expect.objectContaining({ + webSocketMaxConcurrentSends: 4, + webSocketSendRateLimit: 300, + webSocketMaxRecipientConnections: 25, + notificationRecipientConcurrency: 4, + fcmSendConcurrency: 10 + }) + ) + + process.env.MESSAGE_BOX_WEBSOCKET_MAX_CONCURRENT_SENDS = 'unlimited' + process.env.MESSAGE_BOX_WEBSOCKET_SEND_RATE_LIMIT = '-1' + process.env.MESSAGE_BOX_WEBSOCKET_MAX_RECIPIENT_CONNECTIONS = '-1' + expect(readMessageBoxResourceConfig()).toEqual( + expect.objectContaining({ + webSocketMaxConcurrentSends: -1, + webSocketSendRateLimit: -1, + webSocketMaxRecipientConnections: -1 + }) + ) + }) + + it('preserves the deployed MESSAGE_LIST_BATCH_SIZE compatibility setting', () => { + process.env.MESSAGE_LIST_BATCH_SIZE = '750' + expect(readMessageBoxResourceConfig()).toEqual( + expect.objectContaining({ listDefaultLimit: 750, listMaxLimit: 750 }) + ) + }) + + it('prices BRC-105 requests in satoshis and remains disabled by default', () => { + expect(readMessageBoxPricingConfig().enabled).toBe(false) + process.env.MESSAGE_BOX_MONETIZATION_ENABLED = 'true' + const request = { + path: '/sendMessage', + url: '/sendMessage', + body: { + message: { + recipient: ['alice', 'bob'], + body: 'x'.repeat(1024) + } + } + } as unknown as Request + + // 50 base + 10 recipients + 5/KiB + 1 minimum storage satoshi. + expect(calculateConfiguredRequestPrice(request)).toBe(66) + }) + + it('lets operators override a route price explicitly, including free routes', () => { + process.env.MESSAGE_BOX_MONETIZATION_ENABLED = 'true' + process.env.MESSAGE_BOX_ROUTE_PRICES_JSON = JSON.stringify({ '/listMessages': 0 }) + const request = { + path: '/listMessages', + url: '/listMessages', + body: {} + } as unknown as Request + expect(calculateConfiguredRequestPrice(request)).toBe(0) + }) + + it('prices an explicit unlimited-retention policy using its configured prepaid horizon', () => { + process.env.MESSAGE_BOX_MONETIZATION_ENABLED = 'true' + process.env.MESSAGE_BOX_RETENTION_DAYS = '-1' + const request = { + path: '/sendMessage', + url: '/sendMessage', + body: { message: { recipient: 'alice', body: 'x'.repeat(1024 * 1024) } } + } as unknown as Request + + // 50 base + 5 recipient + 5120/KiB + 12000 for 12 MiB-month equivalents. + expect(calculateConfiguredRequestPrice(request)).toBe(17_175) + }) +}) diff --git a/infra/message-box-server/src/config/resources.ts b/infra/message-box-server/src/config/resources.ts new file mode 100644 index 000000000..9eec1c927 --- /dev/null +++ b/infra/message-box-server/src/config/resources.ts @@ -0,0 +1,223 @@ +import { + profileValue, + readResourceLimit, + readResourceProfile, + type ResourceProfileName +} from '../security/edgePolicy.js' + +export interface MessageBoxResourceConfig { + profile: ResourceProfileName + maxMessageBodyBytes: number + maxRecipients: number + listDefaultLimit: number + listMaxLimit: number + listMaxOffset: number + listMaxResponseBytes: number + maxInboxMessages: number + maxInboxBytes: number + maxSenderMessages: number + maxSenderBytes: number + maxAcknowledgmentIds: number + deviceListDefaultLimit: number + deviceListMaxLimit: number + deviceListMaxOffset: number + maxNotificationDevices: number + notificationRecipientConcurrency: number + fcmSendConcurrency: number + webSocketMaxConcurrentSends: number + webSocketSendRateLimit: number + webSocketMaxRecipientConnections: number + permissionListDefaultLimit: number + permissionListMaxLimit: number + permissionListMaxOffset: number + retentionDays: number +} + +function configured( + profile: ResourceProfileName, + suffix: string, + values: { small: number; standard: number; highThroughput: number } +): number { + return readResourceLimit('MESSAGE_BOX', suffix, profileValue(profile, values)) +} + +function legacyListBatchSize(fallback: number): number { + const raw = process.env.MESSAGE_LIST_BATCH_SIZE + if (raw == null || raw.trim() === '') return fallback + const normalized = raw.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error('MESSAGE_LIST_BATCH_SIZE must be -1, unlimited, or a positive integer') + } + const value = Number(normalized) + if (!Number.isSafeInteger(value)) + throw new Error('MESSAGE_LIST_BATCH_SIZE must be a safe integer') + return value +} + +export function readMessageBoxResourceConfig(): MessageBoxResourceConfig { + const profile = readResourceProfile('MESSAGE_BOX') + const config: MessageBoxResourceConfig = { + profile, + maxMessageBodyBytes: configured(profile, 'MAX_MESSAGE_BODY_BYTES', { + small: 256 * 1024, + standard: 1024 * 1024, + highThroughput: 4 * 1024 * 1024 + }), + maxRecipients: configured(profile, 'MAX_RECIPIENTS', { + small: 25, + standard: 100, + highThroughput: 250 + }), + listDefaultLimit: configured(profile, 'LIST_DEFAULT_LIMIT', { + small: legacyListBatchSize(250), + standard: legacyListBatchSize(1_000), + highThroughput: legacyListBatchSize(1_000) + }), + listMaxLimit: configured(profile, 'LIST_MAX_LIMIT', { + small: legacyListBatchSize(500), + standard: legacyListBatchSize(1_000), + highThroughput: legacyListBatchSize(5_000) + }), + listMaxOffset: configured(profile, 'LIST_MAX_OFFSET', { + small: 25_000, + standard: 100_000, + highThroughput: 1_000_000 + }), + listMaxResponseBytes: configured(profile, 'LIST_MAX_RESPONSE_BYTES', { + small: 4 * 1024 * 1024, + standard: 8 * 1024 * 1024, + highThroughput: 32 * 1024 * 1024 + }), + maxInboxMessages: configured(profile, 'MAX_INBOX_MESSAGES', { + small: 5_000, + standard: 10_000, + highThroughput: 100_000 + }), + maxInboxBytes: configured(profile, 'MAX_INBOX_BYTES', { + small: 256 * 1024 * 1024, + standard: 1024 * 1024 * 1024, + highThroughput: 16 * 1024 * 1024 * 1024 + }), + maxSenderMessages: configured(profile, 'MAX_SENDER_MESSAGES', { + small: 5_000, + standard: 10_000, + highThroughput: 100_000 + }), + maxSenderBytes: configured(profile, 'MAX_SENDER_BYTES', { + small: 256 * 1024 * 1024, + standard: 1024 * 1024 * 1024, + highThroughput: 16 * 1024 * 1024 * 1024 + }), + maxAcknowledgmentIds: configured(profile, 'MAX_ACKNOWLEDGMENT_IDS', { + small: 500, + standard: 1_000, + highThroughput: 5_000 + }), + deviceListDefaultLimit: configured(profile, 'DEVICE_LIST_DEFAULT_LIMIT', { + small: 50, + standard: 100, + highThroughput: 500 + }), + deviceListMaxLimit: configured(profile, 'DEVICE_LIST_MAX_LIMIT', { + small: 100, + standard: 100, + highThroughput: 1_000 + }), + deviceListMaxOffset: configured(profile, 'DEVICE_LIST_MAX_OFFSET', { + small: 10_000, + standard: 100_000, + highThroughput: 1_000_000 + }), + maxNotificationDevices: configured(profile, 'MAX_NOTIFICATION_DEVICES', { + small: 25, + standard: 100, + highThroughput: 500 + }), + notificationRecipientConcurrency: configured(profile, 'NOTIFICATION_RECIPIENT_CONCURRENCY', { + small: 2, + standard: 4, + highThroughput: 16 + }), + fcmSendConcurrency: configured(profile, 'FCM_SEND_CONCURRENCY', { + small: 4, + standard: 10, + highThroughput: 50 + }), + webSocketMaxConcurrentSends: configured(profile, 'WEBSOCKET_MAX_CONCURRENT_SENDS', { + small: 2, + standard: 4, + highThroughput: 16 + }), + webSocketSendRateLimit: configured(profile, 'WEBSOCKET_SEND_RATE_LIMIT', { + small: 60, + standard: 300, + highThroughput: 1_200 + }), + webSocketMaxRecipientConnections: configured(profile, 'WEBSOCKET_MAX_RECIPIENT_CONNECTIONS', { + small: 10, + standard: 25, + highThroughput: 100 + }), + permissionListDefaultLimit: configured(profile, 'PERMISSION_LIST_DEFAULT_LIMIT', { + small: 50, + standard: 100, + highThroughput: 500 + }), + permissionListMaxLimit: configured(profile, 'PERMISSION_LIST_MAX_LIMIT', { + small: 100, + standard: 100, + highThroughput: 1_000 + }), + permissionListMaxOffset: configured(profile, 'PERMISSION_LIST_MAX_OFFSET', { + small: 10_000, + standard: 100_000, + highThroughput: 1_000_000 + }), + retentionDays: configured(profile, 'RETENTION_DAYS', { + small: 14, + standard: 30, + highThroughput: 90 + }) + } + + if ( + config.listMaxLimit !== -1 && + config.listDefaultLimit !== -1 && + config.listDefaultLimit > config.listMaxLimit + ) { + throw new Error('MESSAGE_BOX_LIST_DEFAULT_LIMIT must not exceed MESSAGE_BOX_LIST_MAX_LIMIT') + } + if ( + config.deviceListDefaultLimit !== -1 && + config.deviceListMaxLimit !== -1 && + config.deviceListDefaultLimit > config.deviceListMaxLimit + ) { + throw new Error( + 'MESSAGE_BOX_DEVICE_LIST_DEFAULT_LIMIT must not exceed MESSAGE_BOX_DEVICE_LIST_MAX_LIMIT' + ) + } + if ( + config.permissionListDefaultLimit !== -1 && + config.permissionListMaxLimit !== -1 && + config.permissionListDefaultLimit > config.permissionListMaxLimit + ) { + throw new Error( + 'MESSAGE_BOX_PERMISSION_LIST_DEFAULT_LIMIT must not exceed MESSAGE_BOX_PERMISSION_LIST_MAX_LIMIT' + ) + } + return config +} + +export function listQueryBatchSize(config: MessageBoxResourceConfig): number { + if (config.listMaxResponseBytes === -1 || config.maxMessageBodyBytes === -1) return 1 + return Math.max( + 1, + Math.min(100, Math.floor(config.listMaxResponseBytes / config.maxMessageBodyBytes)) + ) +} + +export function messageExpiresAt(config: MessageBoxResourceConfig, now = new Date()): Date | null { + if (config.retentionDays === -1) return null + return new Date(now.getTime() + config.retentionDays * 24 * 60 * 60 * 1_000) +} diff --git a/infra/message-box-server/src/context.ts b/infra/message-box-server/src/context.ts index 70c5376cf..f4b18b7c8 100644 --- a/infra/message-box-server/src/context.ts +++ b/infra/message-box-server/src/context.ts @@ -1,6 +1,7 @@ import type { Knex } from 'knex' -import type { WalletInterface } from '@bsv/sdk' +import type { AsyncSessionManager, SessionManager, WalletInterface } from '@bsv/sdk' import type { Request } from 'express' +import type { PaymentReplayStore } from '@bsv/payment-express-middleware' export interface MessageBoxContext { wallet: WalletInterface @@ -9,6 +10,8 @@ export interface MessageBoxContext { enableWebSockets: boolean enableSwagger: boolean calculateRequestPrice: (req: Request) => Promise | number + sessionManager?: SessionManager | AsyncSessionManager + paymentReplayStore?: PaymentReplayStore logger: Console } @@ -19,6 +22,8 @@ export interface CreateMessageBoxContextOptions { enableWebSockets?: boolean enableSwagger?: boolean calculateRequestPrice?: (req: Request) => Promise | number + sessionManager?: SessionManager | AsyncSessionManager + paymentReplayStore?: PaymentReplayStore logger?: Console } @@ -44,6 +49,8 @@ export function createMessageBoxContext(deps: CreateMessageBoxContextOptions): M } return 0 }), + sessionManager: deps.sessionManager, + paymentReplayStore: deps.paymentReplayStore, logger: deps.logger ?? console } } diff --git a/infra/message-box-server/src/index.ts b/infra/message-box-server/src/index.ts index f9fae4714..63eaaf06c 100644 --- a/infra/message-box-server/src/index.ts +++ b/infra/message-box-server/src/index.ts @@ -17,7 +17,7 @@ */ import * as dotenv from 'dotenv' -import { app, appReady, getWallet, knex } from './app.js' +import { app, appReady, getWallet, knex, paymentReplayStore, sessionManager } from './app.js' import { createServer } from 'node:http' import { Logger, log } from './utils/logger.js' import { trace, SpanStatusCode } from '@opentelemetry/api' @@ -31,6 +31,10 @@ import * as crypto from 'node:crypto' import { initializeFirebase } from './config/firebase.js' import { configureHttpServer } from './security/edgePolicy.js' import { resolveHttpPort } from './config/runtime.js' +import { + startMessageBoxMaintenance, + type MessageBoxMaintenance +} from './security/resourceMaintenance.js' ;(global.self as any) = { crypto } dotenv.config() @@ -75,6 +79,7 @@ configureHttpServer(http, 'MESSAGE_BOX', { // WebSocket setup (only if enabled) // Held in a const container so the exported binding is never reassigned. const ioRef: { current: AuthSocketServer | null } = { current: null } +let maintenance: MessageBoxMaintenance | undefined /** * @function start @@ -98,7 +103,9 @@ export const start = async (): Promise => { const ctx = createMessageBoxContext({ wallet, knex, - enableWebSockets: true + enableWebSockets: true, + sessionManager, + paymentReplayStore }) ioRef.current = attachMessageBoxWebSockets(http, ctx) } @@ -132,6 +139,7 @@ export async function startStandalone(): Promise { }) await start() + maintenance = startMessageBoxMaintenance(knex) await new Promise((resolve, reject) => { http.once('error', reject) http.listen(HTTP_PORT, () => { @@ -147,6 +155,8 @@ let shutdownPromise: Promise | undefined export function shutdownStandalone(signal: NodeJS.Signals): Promise { shutdownPromise ??= (async () => { log.info({ operation: 'server.shutdown', signal }, 'MessageBox shutdown started') + maintenance?.stop() + maintenance = undefined await closeMessageBoxWebSockets(ioRef.current) ioRef.current = null if (http.listening) { diff --git a/infra/message-box-server/src/migrations/2026-08-04-001-resource-safety.ts b/infra/message-box-server/src/migrations/2026-08-04-001-resource-safety.ts new file mode 100644 index 000000000..d17f0610e --- /dev/null +++ b/infra/message-box-server/src/migrations/2026-08-04-001-resource-safety.ts @@ -0,0 +1,43 @@ +import type { Knex } from 'knex' + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('messages', table => { + table.timestamp('expires_at').nullable() + table.index(['expires_at', 'messageId'], 'messages_expiration_index') + }) + + await knex.schema.createTable('message_resource_locks', table => { + table.string('identity_key', 255).primary() + table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now()) + }) + + await knex.schema.createTable('auth_sessions', table => { + // Auth nonces are intentionally bounded by the middleware. Keeping the + // indexed value at 255 characters also stays within conservative MySQL + // utf8mb4 primary-key limits. + table.string('sessionNonce', 255).primary() + table.string('peerNonce', 1024).nullable() + table.string('peerIdentityKey', 255).nullable().index() + table.boolean('isAuthenticated').notNullable().defaultTo(false) + table.bigInteger('lastUpdate').notNullable() + table.boolean('certificatesRequired').nullable() + table.boolean('certificatesValidated').nullable() + table.bigInteger('expiresAt').notNullable().index() + }) + + await knex.schema.createTable('payment_replays', table => { + table.string('transaction_id', 64).primary() + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()) + table.timestamp('expires_at').nullable().index() + }) +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('payment_replays') + await knex.schema.dropTableIfExists('auth_sessions') + await knex.schema.dropTableIfExists('message_resource_locks') + await knex.schema.alterTable('messages', table => { + table.dropIndex(['expires_at', 'messageId'], 'messages_expiration_index') + table.dropColumn('expires_at') + }) +} diff --git a/infra/message-box-server/src/migrations/__tests/resourceSafety.test.ts b/infra/message-box-server/src/migrations/__tests/resourceSafety.test.ts new file mode 100644 index 000000000..153d81ab5 --- /dev/null +++ b/infra/message-box-server/src/migrations/__tests/resourceSafety.test.ts @@ -0,0 +1,39 @@ +import knexFactory, { type Knex } from 'knex' +import { down, up } from '../2026-08-04-001-resource-safety.js' +import { KnexPaymentReplayStore } from '../../security/KnexPaymentReplayStore.js' + +describe('Message Box resource safety migration', () => { + let database: Knex + + beforeEach(async () => { + database = knexFactory({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + await database.schema.createTable('messages', table => { + table.string('messageId').primary() + table.string('body').notNullable() + }) + }) + + afterEach(async () => { + await database.destroy() + }) + + it('creates durable shared-state tables and reverses cleanly', async () => { + await up(database) + await expect(database.schema.hasColumn('messages', 'expires_at')).resolves.toBe(true) + await expect(database.schema.hasTable('message_resource_locks')).resolves.toBe(true) + await expect(database.schema.hasTable('auth_sessions')).resolves.toBe(true) + await expect(database.schema.hasTable('payment_replays')).resolves.toBe(true) + + const replayStore = new KnexPaymentReplayStore(database, 1) + await expect(replayStore.claim('txid')).resolves.toBe(true) + await expect(replayStore.claim('txid')).resolves.toBe(false) + + await down(database) + await expect(database.schema.hasColumn('messages', 'expires_at')).resolves.toBe(false) + await expect(database.schema.hasTable('auth_sessions')).resolves.toBe(false) + }) +}) diff --git a/infra/message-box-server/src/routes/__tests/health.test.ts b/infra/message-box-server/src/routes/__tests/health.test.ts index 872fd39d2..22ce03df0 100644 --- a/infra/message-box-server/src/routes/__tests/health.test.ts +++ b/infra/message-box-server/src/routes/__tests/health.test.ts @@ -23,7 +23,13 @@ describe('public health routes', () => { expect(response.setHeader).toHaveBeenCalledWith('Cache-Control', 'no-store') expect(response.status).toHaveBeenCalledWith(200) - expect(response.json).toHaveBeenCalledWith({ status: 'ok' }) + expect(response.json).toHaveBeenCalledWith({ + ok: true, + status: 'ok', + service: 'messagebox-server', + network: process.env.BSV_NETWORK ?? 'mainnet', + websockets: process.env.ENABLE_WEBSOCKETS !== 'false' + }) }) test('reports readiness when the database responds', async () => { diff --git a/infra/message-box-server/src/routes/__tests/listMessages.test.ts b/infra/message-box-server/src/routes/__tests/listMessages.test.ts index bdb488a4a..9a0ef8dab 100644 --- a/infra/message-box-server/src/routes/__tests/listMessages.test.ts +++ b/infra/message-box-server/src/routes/__tests/listMessages.test.ts @@ -290,6 +290,39 @@ describe('listMessages', () => { ) }) + it('detects another page when the query batch exactly fills the requested limit', async () => { + validReq.body.limit = 8 + const page = Array.from({ length: 8 }, (_, index) => ({ + ...validMessages[0], + messageId: `msg-${index + 1}` + })) + const extra = { ...validMessages[0], messageId: 'msg-9' } + + queryTracker.on('query', (q, sequence) => { + if (sequence === 1) q.response([{ messageBoxId: 123 }]) + else if (sequence === 2) q.response(page) + else if (sequence === 3) q.response([extra]) + else q.response([]) + }) + + await listMessages.func(validReq, mockRes as Response) + + expect(mockRes.status).toHaveBeenCalledWith(200) + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'success', + messages: expect.arrayContaining([ + expect.objectContaining({ messageId: 'msg-1' }), + expect.objectContaining({ messageId: 'msg-8' }) + ]), + limit: 8, + offset: 0, + nextOffset: 8, + hasMore: true + }) + ) + }) + it('Throws unknown errors', async () => { queryTracker.on('query', () => { throw new Error('Failed') diff --git a/infra/message-box-server/src/routes/__tests/sendMessage.test.ts b/infra/message-box-server/src/routes/__tests/sendMessage.test.ts index 71a4da200..c4bee64e6 100644 --- a/infra/message-box-server/src/routes/__tests/sendMessage.test.ts +++ b/infra/message-box-server/src/routes/__tests/sendMessage.test.ts @@ -53,7 +53,20 @@ const mockRes: jest.Mocked = { let validReq: SendMessageRequest // eslint-disable-next-line @typescript-eslint/no-unused-vars let validRes: { status: string } -let validMessageBox: { messageBoxId: number; type: string } +function successfulStoreResponse(q: { sql: string; response: (value: unknown) => void }): void { + if (q.sql.includes('select `identityKey`, `messageBoxId` from `messageBox`')) { + q.response([ + { + identityKey: '028d37b941208cd6b8a4c28288eda5f2f16c2b3ab0fcb6d13c18b47fe37b971fc1', + messageBoxId: 42 + } + ]) + } else if (q.sql.includes('message_count') && q.sql.includes('body_bytes')) { + q.response([{ message_count: 0, body_bytes: 0 }]) + } else { + q.response([]) + } +} describe('sendMessage', () => { // Capture original console methods @@ -84,11 +97,6 @@ describe('sendMessage', () => { validRes = { status: 'success' } - validMessageBox = { - messageBoxId: 42, - type: 'payment_inbox' - } - validReq = { auth: { identityKey: 'mockIdKey' @@ -107,6 +115,10 @@ describe('sendMessage', () => { }) afterEach(() => { + delete process.env.MESSAGE_BOX_MAX_SENDER_MESSAGES + delete process.env.MESSAGE_BOX_MAX_SENDER_BYTES + delete process.env.MESSAGE_BOX_MAX_INBOX_MESSAGES + delete process.env.MESSAGE_BOX_MAX_INBOX_BYTES jest.clearAllMocks() if (queryTracker !== null && queryTracker !== undefined) { @@ -328,17 +340,7 @@ describe('sendMessage', () => { }) it('Creates a messageBox when it does not exist', async () => { - queryTracker.on('query', (q, step: number) => { - if (step === 1) { - q.response(0) // Simulate that the messageBox does not exist - } else if (step === 2) { - q.response([validMessageBox]) // Simulate messageBox being inserted - } else if (step === 3) { - q.response([validMessageBox]) // Simulate finding a valid messageBoxId - } else { - q.response([]) // Default response - } - }) + queryTracker.on('query', q => successfulStoreResponse(q)) await sendMessage.func(validReq, mockRes as Response) @@ -350,25 +352,46 @@ describe('sendMessage', () => { ) }) - it('Silently ignores duplicate messages via onConflict().ignore()', async () => { - queryTracker.on('query', (q, step: number) => { - if (step === 1) { - q.response({ messageBoxId: 42, type: 'payment_inbox' }) // messageBox exists - } else if (step === 2) { - q.response({ messageBoxId: 42 }) // get messageBoxId for insert - } else if (step === 3) { - q.response(0) // insert with onConflict().ignore() returns 0 rows affected - } else { - q.response([]) + it('rejects a duplicate message and rolls the transaction back', async () => { + queryTracker.on('query', q => { + if (q.sql.startsWith('insert into `messages`')) { + const error = Object.assign(new Error('duplicate'), { code: 'ER_DUP_ENTRY' }) + q.reject(error) + return } + successfulStoreResponse(q) }) await sendMessage.func(validReq, mockRes as Response) - expect(mockRes.status).toHaveBeenCalledWith(200) + expect(mockRes.status).toHaveBeenCalledWith(400) expect(mockRes.json).toHaveBeenCalledWith( expect.objectContaining({ - status: 'success' + status: 'error', + code: 'ERR_DUPLICATE_MESSAGE' + }) + ) + }) + + it('rejects storage atomically when the shared sender quota is exhausted', async () => { + process.env.MESSAGE_BOX_MAX_SENDER_MESSAGES = '1' + queryTracker.on('query', q => { + if (q.sql.includes('message_count') && q.sql.includes('where `sender` = ?')) { + q.response([{ message_count: 1, body_bytes: 2 }]) + return + } + successfulStoreResponse(q) + }) + + await sendMessage.func(validReq, mockRes as Response) + + expect(mockRes.status).toHaveBeenCalledWith(429) + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'error', + code: 'ERR_SENDER_QUOTA_EXCEEDED', + resource: 'messages', + limit: 1 }) ) }) @@ -397,17 +420,7 @@ describe('sendMessage', () => { }) it('creates a new messageBox when one does not exist for recipient', async () => { - queryTracker.on('query', (q, step) => { - if (step === 1) - q.response(undefined) // messageBox not found - else if (step === 2) - q.response(1) // messageBox insert - else if (step === 3) - q.response({ messageBoxId: 42 }) // get messageBoxId for message insert - else if (step === 4) - q.response(1) // insert message - else q.response([]) - }) + queryTracker.on('query', q => successfulStoreResponse(q)) await sendMessage.func(validReq, mockRes) diff --git a/infra/message-box-server/src/routes/acknowledgeMessage.ts b/infra/message-box-server/src/routes/acknowledgeMessage.ts index 695f85d7f..bc5ab6593 100644 --- a/infra/message-box-server/src/routes/acknowledgeMessage.ts +++ b/infra/message-box-server/src/routes/acknowledgeMessage.ts @@ -13,6 +13,7 @@ import { Response } from 'express' import { AuthRequest } from '@bsv/auth-express-middleware' import { Logger } from '../utils/logger.js' import { runtimeDeps } from '../runtimeDeps.js' +import { readMessageBoxResourceConfig } from '../config/resources.js' export const MAX_ACKNOWLEDGMENT_IDS = 1_000 export const MAX_MESSAGE_ID_BYTES = 256 @@ -120,6 +121,7 @@ export default { Array.isArray(messageIds) ? messageIds.length : 0, 'message(s)' ) + const maxAcknowledgmentIds = readMessageBoxResourceConfig().maxAcknowledgmentIds // Validate request: must be a non-empty array of strings if (messageIds == null || (Array.isArray(messageIds) && messageIds.length === 0)) { @@ -132,7 +134,7 @@ export default { if ( !Array.isArray(messageIds) || - messageIds.length > MAX_ACKNOWLEDGMENT_IDS || + (maxAcknowledgmentIds !== -1 && messageIds.length > maxAcknowledgmentIds) || messageIds.some( id => typeof id !== 'string' || @@ -140,11 +142,12 @@ export default { Buffer.byteLength(id, 'utf8') > MAX_MESSAGE_ID_BYTES ) ) { + const maximumIds = maxAcknowledgmentIds === -1 ? '' : ` of at most ${maxAcknowledgmentIds}` return res.status(400).json({ status: 'error', code: 'ERR_INVALID_MESSAGE_ID', description: - `Message IDs must be a non-empty array of at most ${MAX_ACKNOWLEDGMENT_IDS} ` + + `Message IDs must be a non-empty array${maximumIds} ` + `non-empty strings no longer than ${MAX_MESSAGE_ID_BYTES} bytes each.` }) } diff --git a/infra/message-box-server/src/routes/health.ts b/infra/message-box-server/src/routes/health.ts index 9a51096c3..4705557d2 100644 --- a/infra/message-box-server/src/routes/health.ts +++ b/infra/message-box-server/src/routes/health.ts @@ -8,10 +8,21 @@ export const healthRoute = { path: '/health', func: (_req: Request, res: Response): Response => { res.setHeader('Cache-Control', NO_STORE) - return res.status(200).json({ status: 'ok' }) + return res.status(200).json({ + ok: true, + status: 'ok', + service: 'messagebox-server', + network: process.env.BSV_NETWORK ?? 'mainnet', + websockets: process.env.ENABLE_WEBSOCKETS !== 'false' + }) } } +export const healthzRoute = { + ...healthRoute, + path: '/healthz' +} + export const readinessRoute = { type: 'get', path: '/ready', diff --git a/infra/message-box-server/src/routes/index.ts b/infra/message-box-server/src/routes/index.ts index d71be7bca..11a33f1bd 100644 --- a/infra/message-box-server/src/routes/index.ts +++ b/infra/message-box-server/src/routes/index.ts @@ -4,11 +4,12 @@ import acknowledgeMessage from './acknowledgeMessage.js' import registerDevice from './registerDevice.js' import listDevices from './listDevices.js' import { permissionRoutes } from './permissions/index.js' -import { healthRoute, readinessRoute } from './health.js' +import { healthRoute, healthzRoute, readinessRoute } from './health.js' // Explicitly type the exported arrays to avoid type inference issues export const preAuth: Array<{ type: string; path: string; func: Function }> = [ healthRoute, + healthzRoute, readinessRoute ] export const postAuth: Array<{ type: string; path: string; func: Function }> = [ diff --git a/infra/message-box-server/src/routes/listDevices.ts b/infra/message-box-server/src/routes/listDevices.ts index 8154a2d50..daecf0b8f 100644 --- a/infra/message-box-server/src/routes/listDevices.ts +++ b/infra/message-box-server/src/routes/listDevices.ts @@ -2,6 +2,7 @@ import { Response } from 'express' import { Logger } from '../utils/logger.js' import { AuthRequest } from '@bsv/auth-express-middleware' import { runtimeDeps } from '../runtimeDeps.js' +import { readMessageBoxResourceConfig } from '../config/resources.js' export const MAX_DEVICE_PAGE_SIZE = 100 export const MAX_DEVICE_OFFSET = 100_000 @@ -42,22 +43,43 @@ export default { }) } + const resources = readMessageBoxResourceConfig() const limitValue = req.query?.limit const offsetValue = req.query?.offset - const limit = limitValue == null ? MAX_DEVICE_PAGE_SIZE : Number(limitValue) + let limit = Number(limitValue) + if (limitValue == null) { + limit = + resources.deviceListDefaultLimit === -1 + ? Number.MAX_SAFE_INTEGER + : resources.deviceListDefaultLimit + } const offset = offsetValue == null ? 0 : Number(offsetValue) - if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_DEVICE_PAGE_SIZE) { + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + (resources.deviceListMaxLimit !== -1 && limit > resources.deviceListMaxLimit) + ) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_LIMIT', - description: `limit must be an integer between 1 and ${MAX_DEVICE_PAGE_SIZE}.` + description: + resources.deviceListMaxLimit === -1 + ? 'limit must be a positive safe integer.' + : `limit must be an integer between 1 and ${resources.deviceListMaxLimit}.` }) } - if (!Number.isSafeInteger(offset) || offset < 0 || offset > MAX_DEVICE_OFFSET) { + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + (resources.deviceListMaxOffset !== -1 && offset > resources.deviceListMaxOffset) + ) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_OFFSET', - description: `offset must be an integer between 0 and ${MAX_DEVICE_OFFSET}.` + description: + resources.deviceListMaxOffset === -1 + ? 'offset must be a non-negative safe integer.' + : `offset must be an integer between 0 and ${resources.deviceListMaxOffset}.` }) } diff --git a/infra/message-box-server/src/routes/listMessages.ts b/infra/message-box-server/src/routes/listMessages.ts index a8f7f591f..686980329 100644 --- a/infra/message-box-server/src/routes/listMessages.ts +++ b/infra/message-box-server/src/routes/listMessages.ts @@ -13,6 +13,11 @@ import { Response } from 'express' import { AuthRequest } from '@bsv/auth-express-middleware' import { log } from '../utils/logger.js' import { runtimeDeps } from '../runtimeDeps.js' +import { + listQueryBatchSize, + readMessageBoxResourceConfig, + type MessageBoxResourceConfig +} from '../config/resources.js' export const MAX_LIST_MESSAGE_BOX_BYTES = 128 export const MAX_LIST_MESSAGES_PAGE_SIZE = 1_000 @@ -28,6 +33,197 @@ interface ListMessagesRequest extends AuthRequest { messageBox?: string limit?: number offset?: number + skip?: number + } +} + +interface RouteFailure { + statusCode: number + code: string + description: string +} + +interface ListPagination { + limit: number + offset: number +} + +interface MessagePage extends ListPagination { + messages: Array> + nextOffset: number + hasMore: boolean +} + +interface PageAccumulator { + messages: Array> + encodedBytes: number + queryOffset: number +} + +function routeFailure(statusCode: number, code: string, description: string): RouteFailure { + return { statusCode, code, description } +} + +function isRouteFailure(value: unknown): value is RouteFailure { + return typeof value === 'object' && value != null && 'statusCode' in value +} + +function normalizeMessageBoxName(value: unknown): string | RouteFailure { + if (value == null || (typeof value === 'string' && value.trim() === '')) { + return routeFailure( + 400, + 'ERR_MESSAGEBOX_REQUIRED', + 'Please provide the name of a valid MessageBox!' + ) + } + if (typeof value !== 'string') { + return routeFailure(400, 'ERR_INVALID_MESSAGEBOX', 'MessageBox name must be a string!') + } + const normalized = value.trim() + if (Buffer.byteLength(normalized, 'utf8') > MAX_LIST_MESSAGE_BOX_BYTES) { + return routeFailure( + 400, + 'ERR_INVALID_MESSAGEBOX', + `MessageBox names must not exceed ${MAX_LIST_MESSAGE_BOX_BYTES} bytes.` + ) + } + return normalized +} + +function isBoundedInteger(value: number, minimum: number, maximum: number): boolean { + return Number.isSafeInteger(value) && value >= minimum && (maximum === -1 || value <= maximum) +} + +function parseListPagination( + body: ListMessagesRequest['body'], + resources: MessageBoxResourceConfig +): ListPagination | RouteFailure { + if (body.offset != null && body.skip != null && body.offset !== body.skip) { + return routeFailure( + 400, + 'ERR_INVALID_OFFSET', + 'offset and skip must match when both are provided.' + ) + } + const configuredDefault = + resources.listDefaultLimit === -1 ? Number.MAX_SAFE_INTEGER : resources.listDefaultLimit + const limit = body.limit ?? configuredDefault + const offset = body.offset ?? body.skip ?? 0 + if (!isBoundedInteger(limit, 1, resources.listMaxLimit)) { + const maximum = + resources.listMaxLimit === -1 + ? 'the JavaScript safe-integer maximum' + : String(resources.listMaxLimit) + return routeFailure( + 400, + 'ERR_INVALID_LIMIT', + `limit must be an integer between 1 and ${maximum}.` + ) + } + if (!isBoundedInteger(offset, 0, resources.listMaxOffset)) { + const maximum = + resources.listMaxOffset === -1 + ? 'the JavaScript safe-integer maximum' + : String(resources.listMaxOffset) + return routeFailure( + 400, + 'ERR_INVALID_OFFSET', + `offset must be an integer between 0 and ${maximum}.` + ) + } + return { limit, offset } +} + +function appendMessage( + accumulator: PageAccumulator, + message: Record, + maxResponseBytes: number +): 'added' | 'full' | 'oversized' { + const formatted = { + messageId: message.messageId, + body: typeof message.body === 'string' ? message.body : JSON.stringify(message.body), + sender: message.sender, + createdAt: message.created_at, + updatedAt: message.updated_at + } + const itemBytes = Buffer.byteLength(JSON.stringify(formatted), 'utf8') + 1 + if (maxResponseBytes !== -1 && accumulator.encodedBytes + itemBytes > maxResponseBytes) { + return accumulator.messages.length === 0 ? 'oversized' : 'full' + } + accumulator.messages.push(formatted) + accumulator.encodedBytes += itemBytes + accumulator.queryOffset += 1 + return 'added' +} + +function appendMessageRows( + accumulator: PageAccumulator, + messageRows: Array>, + pagination: ListPagination, + maxResponseBytes: number +): boolean | RouteFailure { + for (const message of messageRows) { + if (accumulator.messages.length >= pagination.limit) return true + const outcome = appendMessage(accumulator, message, maxResponseBytes) + if (outcome === 'oversized') { + return routeFailure( + 413, + 'ERR_MESSAGE_RESPONSE_TOO_LARGE', + 'The oldest message exceeds the configured listing response budget.' + ) + } + if (outcome === 'full') return true + } + return false +} + +async function readMessagePage( + identityKey: string, + messageBoxId: number, + pagination: ListPagination, + resources: MessageBoxResourceConfig +): Promise { + const accumulator: PageAccumulator = { + messages: [], + encodedBytes: 256, + queryOffset: pagination.offset + } + const batchSize = listQueryBatchSize(resources) + let hasMore = false + + while (accumulator.messages.length <= pagination.limit) { + const remaining = pagination.limit - accumulator.messages.length + const take = Math.max(1, Math.min(batchSize, remaining + 1)) + const messageRows = await runtimeDeps + .knex('messages') + .where({ recipient: identityKey, messageBoxId }) + .where(function () { + this.whereNull('expires_at').orWhere('expires_at', '>', new Date()) + }) + .select('messageId', 'body', 'sender', 'created_at', 'updated_at') + .orderBy('created_at', 'asc') + .orderBy('messageId', 'asc') + .limit(take) + .offset(accumulator.queryOffset) + + if (messageRows.length === 0) break + const appendResult = appendMessageRows( + accumulator, + messageRows, + pagination, + resources.listMaxResponseBytes + ) + if (isRouteFailure(appendResult)) return appendResult + hasMore = appendResult + if (hasMore || messageRows.length < take) break + } + + return { + messages: accumulator.messages, + limit: pagination.limit, + offset: pagination.offset, + nextOffset: accumulator.queryOffset, + hasMore } } @@ -52,6 +248,18 @@ interface ListMessagesRequest extends AuthRequest { * messageBox: * type: string * description: The name of the messageBox to retrieve messages from + * limit: + * type: integer + * minimum: 1 + * default: 1000 + * offset: + * type: integer + * minimum: 0 + * default: 0 + * skip: + * type: integer + * minimum: 0 + * description: Compatibility alias for offset * responses: * 200: * description: Successfully retrieved messages (can be empty) @@ -80,6 +288,14 @@ interface ListMessagesRequest extends AuthRequest { * updatedAt: * type: string * format: date-time + * limit: + * type: integer + * offset: + * type: integer + * nextOffset: + * type: integer + * hasMore: + * type: boolean * 400: * description: Invalid or missing messageBox name * 500: @@ -137,9 +353,7 @@ export default { */ func: async (req: ListMessagesRequest, res: Response): Promise => { try { - const { messageBox } = req.body const identityKey = req.auth?.identityKey - if (identityKey == null || identityKey.trim() === '') { return res.status(401).json({ status: 'error', @@ -148,50 +362,24 @@ export default { }) } - // Validate a messageBox is provided and is a string - if (messageBox == null || (typeof messageBox === 'string' && messageBox.trim() === '')) { - return res.status(400).json({ - status: 'error', - code: 'ERR_MESSAGEBOX_REQUIRED', - description: 'Please provide the name of a valid MessageBox!' - }) - } - - if (typeof messageBox !== 'string') { - return res.status(400).json({ + const normalizedMessageBox = normalizeMessageBoxName(req.body.messageBox) + if (isRouteFailure(normalizedMessageBox)) { + return res.status(normalizedMessageBox.statusCode).json({ status: 'error', - code: 'ERR_INVALID_MESSAGEBOX', - description: 'MessageBox name must be a string!' + code: normalizedMessageBox.code, + description: normalizedMessageBox.description }) } - - const normalizedMessageBox = messageBox.trim() - if (Buffer.byteLength(normalizedMessageBox, 'utf8') > MAX_LIST_MESSAGE_BOX_BYTES) { - return res.status(400).json({ - status: 'error', - code: 'ERR_INVALID_MESSAGEBOX', - description: `MessageBox names must not exceed ${MAX_LIST_MESSAGE_BOX_BYTES} bytes.` - }) - } - - const limit = req.body.limit ?? MAX_LIST_MESSAGES_PAGE_SIZE - const offset = req.body.offset ?? 0 - if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST_MESSAGES_PAGE_SIZE) { - return res.status(400).json({ - status: 'error', - code: 'ERR_INVALID_LIMIT', - description: `limit must be an integer between 1 and ${MAX_LIST_MESSAGES_PAGE_SIZE}.` - }) - } - if (!Number.isSafeInteger(offset) || offset < 0 || offset > MAX_LIST_MESSAGES_OFFSET) { - return res.status(400).json({ + const resourceConfig = readMessageBoxResourceConfig() + const pagination = parseListPagination(req.body, resourceConfig) + if (isRouteFailure(pagination)) { + return res.status(pagination.statusCode).json({ status: 'error', - code: 'ERR_INVALID_OFFSET', - description: `offset must be an integer between 0 and ${MAX_LIST_MESSAGES_OFFSET}.` + code: pagination.code, + description: pagination.description }) } - // Find the messageBox ID for this user const [messageBoxRecord] = await runtimeDeps .knex('messageBox') .where({ @@ -200,50 +388,30 @@ export default { }) .select('messageBoxId') - // Return empty array if no messageBox was found if (messageBoxRecord === undefined) { return res.status(200).json({ status: 'success', messages: [], - limit, - offset, + ...pagination, + nextOffset: pagination.offset, hasMore: false }) } - // Retrieve one bounded, deterministic page. - const messageRows = await runtimeDeps - .knex('messages') - .where({ - recipient: identityKey, - messageBoxId: messageBoxRecord.messageBoxId + const page = await readMessagePage( + identityKey, + messageBoxRecord.messageBoxId, + pagination, + resourceConfig + ) + if (isRouteFailure(page)) { + return res.status(page.statusCode).json({ + status: 'error', + code: page.code, + description: page.description }) - .select('messageId', 'body', 'sender', 'created_at', 'updated_at') - .orderBy('created_at', 'asc') - .orderBy('messageId', 'asc') - .limit(limit + 1) - .offset(offset) - - const hasMore = messageRows.length > limit - const messages = messageRows.slice(0, limit) - - // Normalize all message bodies to strings and convert to camelCase - const formattedMessages = messages.map(message => ({ - messageId: message.messageId, - body: typeof message.body === 'string' ? message.body : JSON.stringify(message.body), - sender: message.sender, - createdAt: message.created_at, - updatedAt: message.updated_at - })) - - // Return a list of matching messages - return res.status(200).json({ - status: 'success', - messages: formattedMessages, - limit, - offset, - hasMore - }) + } + return res.status(200).json({ status: 'success', ...page }) } catch (e) { log.error({ operation: 'messages.list', outcome: 'error', err: e }, 'Failed to list messages') return res.status(500).json({ diff --git a/infra/message-box-server/src/routes/permissions/listPermissions.ts b/infra/message-box-server/src/routes/permissions/listPermissions.ts index 12be66eb1..b8d848e1b 100644 --- a/infra/message-box-server/src/routes/permissions/listPermissions.ts +++ b/infra/message-box-server/src/routes/permissions/listPermissions.ts @@ -2,6 +2,10 @@ import { Response } from 'express' import { AuthRequest } from '@bsv/auth-express-middleware' import { Logger } from '../../utils/logger.js' import { runtimeDeps } from '../../runtimeDeps.js' +import { + readMessageBoxResourceConfig, + type MessageBoxResourceConfig +} from '../../config/resources.js' export const MAX_PERMISSION_PAGE_SIZE = 100 export const MAX_PERMISSION_OFFSET = 100_000 @@ -16,6 +20,84 @@ export interface ListPermissionsRequest extends AuthRequest { } } +interface PermissionPagination { + limit: number + offset: number + sortOrder: 'asc' | 'desc' +} + +interface ValidationFailure { + code: string + description: string +} + +function parsePermissionPagination( + query: ListPermissionsRequest['query'], + resources: MessageBoxResourceConfig +): PermissionPagination | ValidationFailure { + let limit = Number(query.limit) + if (query.limit == null) { + limit = + resources.permissionListDefaultLimit === -1 + ? Number.MAX_SAFE_INTEGER + : resources.permissionListDefaultLimit + } + const offset = query.offset == null ? 0 : Number(query.offset) + const sortOrder = query.createdAtOrder ?? 'desc' + + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + (resources.permissionListMaxLimit !== -1 && limit > resources.permissionListMaxLimit) + ) { + const description = + resources.permissionListMaxLimit === -1 + ? 'limit must be a positive safe integer.' + : `limit must be an integer between 1 and ${resources.permissionListMaxLimit}.` + return { code: 'ERR_INVALID_LIMIT', description } + } + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + (resources.permissionListMaxOffset !== -1 && offset > resources.permissionListMaxOffset) + ) { + const description = + resources.permissionListMaxOffset === -1 + ? 'offset must be a non-negative safe integer.' + : `offset must be an integer between 0 and ${resources.permissionListMaxOffset}.` + return { code: 'ERR_INVALID_OFFSET', description } + } + if (sortOrder !== 'asc' && sortOrder !== 'desc') { + return { + code: 'ERR_INVALID_SORT_ORDER', + description: 'createdAtOrder must be asc or desc.' + } + } + return { limit, offset, sortOrder } +} + +function normalizeMessageBoxFilter(messageBox: unknown): string | ValidationFailure | undefined { + if (messageBox == null) return undefined + if (typeof messageBox !== 'string') { + return { + code: 'ERR_INVALID_MESSAGE_BOX', + description: `messageBox must be a non-empty string of at most ${MAX_MESSAGE_BOX_BYTES} bytes.` + } + } + const normalized = messageBox.trim() + if (normalized === '' || Buffer.byteLength(normalized, 'utf8') > MAX_MESSAGE_BOX_BYTES) { + return { + code: 'ERR_INVALID_MESSAGE_BOX', + description: `messageBox must be a non-empty string of at most ${MAX_MESSAGE_BOX_BYTES} bytes.` + } + } + return normalized +} + +function isValidationFailure(value: unknown): value is ValidationFailure { + return typeof value === 'object' && value != null && 'code' in value +} + /** * @swagger * /permissions/list: @@ -122,55 +204,31 @@ export default { } // Parse and validate query parameters - const { messageBox, limit: limitStr, offset: offsetStr, createdAtOrder } = req.query - - const limit = limitStr != null ? Number(limitStr) : MAX_PERMISSION_PAGE_SIZE - const offset = offsetStr != null ? Number(offsetStr) : 0 - const sortOrder = createdAtOrder ?? 'desc' - - // Validate pagination parameters - if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_PERMISSION_PAGE_SIZE) { - return res.status(400).json({ - status: 'error', - code: 'ERR_INVALID_LIMIT', - description: `limit must be an integer between 1 and ${MAX_PERMISSION_PAGE_SIZE}.` - }) - } - - if (!Number.isSafeInteger(offset) || offset < 0 || offset > MAX_PERMISSION_OFFSET) { - return res.status(400).json({ - status: 'error', - code: 'ERR_INVALID_OFFSET', - description: `offset must be an integer between 0 and ${MAX_PERMISSION_OFFSET}.` - }) - } - - if (sortOrder !== 'asc' && sortOrder !== 'desc') { + const { messageBox } = req.query + const resources = readMessageBoxResourceConfig() + const pagination = parsePermissionPagination(req.query, resources) + if (isValidationFailure(pagination)) { return res.status(400).json({ status: 'error', - code: 'ERR_INVALID_SORT_ORDER', - description: 'createdAtOrder must be asc or desc.' + code: pagination.code, + description: pagination.description }) } - - if ( - messageBox != null && - (typeof messageBox !== 'string' || - messageBox.trim() === '' || - Buffer.byteLength(messageBox.trim(), 'utf8') > MAX_MESSAGE_BOX_BYTES) - ) { + const normalizedMessageBox = normalizeMessageBoxFilter(messageBox) + if (isValidationFailure(normalizedMessageBox)) { return res.status(400).json({ status: 'error', - code: 'ERR_INVALID_MESSAGE_BOX', - description: `messageBox must be a non-empty string of at most ${MAX_MESSAGE_BOX_BYTES} bytes.` + code: normalizedMessageBox.code, + description: normalizedMessageBox.description }) } + const { limit, offset, sortOrder } = pagination // Validate identity key format const recipientKey = req.auth.identityKey Logger.log( - `[DEBUG] Listing permissions for recipient: ${recipientKey}, messageBox: ${messageBox ?? 'all'}, limit: ${limit}, offset: ${offset}, createdAtOrder: ${sortOrder}` + `[DEBUG] Listing permissions for recipient: ${recipientKey}, messageBox: ${normalizedMessageBox ?? 'all'}, limit: ${limit}, offset: ${offset}, createdAtOrder: ${sortOrder}` ) // Build base query @@ -185,8 +243,8 @@ export default { ]) // Apply messageBox filter if provided - if (messageBox != null) { - query = query.where('message_box', messageBox.trim()) + if (normalizedMessageBox != null) { + query = query.where('message_box', normalizedMessageBox) } // Get total count for pagination info (before applying limit/offset) diff --git a/infra/message-box-server/src/routes/sendMessage.ts b/infra/message-box-server/src/routes/sendMessage.ts index 0cc21fab6..39a7fd52e 100644 --- a/infra/message-box-server/src/routes/sendMessage.ts +++ b/infra/message-box-server/src/routes/sendMessage.ts @@ -35,6 +35,14 @@ import { shouldUseFCMDelivery } from '../utils/messagePermissions.js' import { runtimeDeps, getWallet } from '../runtimeDeps.js' +import { + messageExpiresAt, + readMessageBoxResourceConfig, + type MessageBoxResourceConfig +} from '../config/resources.js' +import { readMessageBoxPricingConfig } from '../config/pricing.js' +import type { Knex } from 'knex' +import { mapWithConcurrency } from '../utils/boundedConcurrency.js' // Type definition for the incoming message format export interface Message { @@ -143,21 +151,30 @@ function validateMessageBox(message: Message): RouteResult { return routeValue(boxType) } -function validateMessageBody(message: Message): RouteResult { +function validateMessageBody( + message: Message, + resourceConfig: MessageBoxResourceConfig +): RouteResult { if (typeof message.body !== 'string' || message.body.trim() === '') { return routeFailure(400, 'ERR_INVALID_MESSAGE_BODY', 'Invalid message body.') } - if (Buffer.byteLength(message.body, 'utf8') > MAX_MESSAGE_BODY_BYTES) { + if ( + resourceConfig.maxMessageBodyBytes !== -1 && + Buffer.byteLength(message.body, 'utf8') > resourceConfig.maxMessageBodyBytes + ) { return routeFailure( 413, 'ERR_MESSAGE_BODY_TOO_LARGE', - `Message bodies must not exceed ${MAX_MESSAGE_BODY_BYTES} bytes.` + `Message bodies must not exceed ${resourceConfig.maxMessageBodyBytes} bytes.` ) } return routeValue(undefined) } -function normalizeRecipients(message: Message): RouteResult { +function normalizeRecipients( + message: Message, + resourceConfig: MessageBoxResourceConfig +): RouteResult { const recipientsRaw: unknown = message.recipients ?? message.recipient if (recipientsRaw == null) { return routeFailure( @@ -167,11 +184,16 @@ function normalizeRecipients(message: Message): RouteResult { ) } const recipients = Array.isArray(recipientsRaw) ? recipientsRaw : [recipientsRaw] - if (recipients.length === 0 || recipients.length > MAX_MESSAGE_RECIPIENTS) { + if ( + recipients.length === 0 || + (resourceConfig.maxRecipients !== -1 && recipients.length > resourceConfig.maxRecipients) + ) { return routeFailure( 400, 'ERR_TOO_MANY_RECIPIENTS', - `A message may include at most ${MAX_MESSAGE_RECIPIENTS} recipients.` + resourceConfig.maxRecipients === -1 + ? 'A message must include at least one recipient.' + : `A message may include at most ${resourceConfig.maxRecipients} recipients.` ) } return routeValue(recipients.map(recipient => String(recipient).trim())) @@ -234,9 +256,10 @@ function validateMessage(message: Message | undefined): RouteResult { - for (const recipient of recipients) { - const existing = await runtimeDeps - .knex('messageBox') - .where({ identityKey: recipient, type: boxType }) - .first() - if (existing == null) { - await runtimeDeps.knex('messageBox').insert({ - identityKey: recipient, - type: boxType, - created_at: new Date(), - updated_at: new Date() - }) - } - } -} - async function evaluateRecipientFees( recipients: string[], senderKey: string, @@ -449,48 +455,128 @@ function hasErrorCode(error: unknown, code: string): boolean { return error != null && typeof error === 'object' && 'code' in error && error.code === code } -async function storeMessage( +function isDuplicateDatabaseError(error: unknown): boolean { + return ( + hasErrorCode(error, 'ER_DUP_ENTRY') || + hasErrorCode(error, 'SQLITE_CONSTRAINT_PRIMARYKEY') || + hasErrorCode(error, 'SQLITE_CONSTRAINT_UNIQUE') + ) +} + +class RouteFailureError extends Error { + constructor(readonly failure: RouteFailure) { + const detail = failure.payload.description ?? failure.payload.code + super(typeof detail === 'string' ? detail : 'Route failure') + } +} + +interface StoredMessageRow { + messageId: string + messageBoxId: number + sender: string + recipient: string + body: string + bodyBytes: number + created_at: Date + updated_at: Date + expires_at: Date | null +} + +interface ResourceUsage { + messageCount: number + bodyBytes: number +} + +function numericAggregate(value: unknown): number { + if (typeof value === 'number' && Number.isFinite(value)) return value + if (typeof value === 'bigint') return Number(value) + if (typeof value === 'string' && /^\d+$/.test(value)) return Number(value) + return 0 +} + +function activeMessages(query: Knex.QueryBuilder, now: Date): Knex.QueryBuilder { + return query.where(builder => { + builder.whereNull('expires_at').orWhere('expires_at', '>', now) + }) +} + +async function resourceUsage( + transaction: Knex.Transaction, + column: 'sender' | 'recipient', + identityKey: string, + now: Date +): Promise { + const byteFunction = transaction.client.config.client.includes('sqlite') + ? 'LENGTH(??)' + : 'OCTET_LENGTH(??)' + const result = await activeMessages(transaction('messages').where(column, identityKey), now) + .count<{ message_count: string | number }[]>({ message_count: '*' }) + .select(transaction.raw(`COALESCE(SUM(${byteFunction}), 0) AS ??`, ['body', 'body_bytes'])) + .first() + return { + messageCount: numericAggregate(result?.message_count), + bodyBytes: numericAggregate((result as Record | undefined)?.body_bytes) + } +} + +function enforceQuota( + usage: ResourceUsage, + additions: ResourceUsage, + maxMessages: number, + maxBytes: number, + code: string, + description: string +): void { + if (maxMessages !== -1 && usage.messageCount + additions.messageCount > maxMessages) { + throw new RouteFailureError( + routeFailure(429, code, description, { + resource: 'messages', + limit: maxMessages + }) + ) + } + if (maxBytes !== -1 && usage.bodyBytes + additions.bodyBytes > maxBytes) { + throw new RouteFailureError( + routeFailure(429, code, description, { + resource: 'bytes', + limit: maxBytes + }) + ) + } +} + +async function acquireResourceLocks( + transaction: Knex.Transaction, + identities: string[], + now: Date +): Promise { + const keys = [...new Set(identities)].sort((left, right) => left.localeCompare(right)) + await transaction('message_resource_locks') + .insert(keys.map(identity_key => ({ identity_key, updated_at: now }))) + .onConflict('identity_key') + .ignore() + // Stable ordering prevents deadlocks when multi-recipient requests overlap. + await transaction('message_resource_locks') + .whereIn('identity_key', keys) + .orderBy('identity_key', 'asc') + .select('identity_key') + .forUpdate() +} + +function buildStoredBody( validated: ValidatedMessage, recipient: string, - messageId: string, - senderKey: string, payment: Payment | undefined, recipientOutputs: RecipientOutputs -): Promise { - const messageBox = await runtimeDeps - .knex('messageBox') - .where({ identityKey: recipient, type: validated.boxType }) - .select('messageBoxId') - .first() +): string { const recipientPayment = recipientOutputs.has(recipient) && payment != null ? { ...payment, outputs: recipientOutputs.get(recipient)! } : undefined - const storedBody = { + return JSON.stringify({ message: validated.message.body, ...(recipientPayment != null ? { payment: recipientPayment } : {}) - } - try { - await runtimeDeps - .knex('messages') - .insert({ - messageId, - messageBoxId: messageBox?.messageBoxId ?? null, - sender: senderKey, - recipient, - body: JSON.stringify(storedBody), - created_at: new Date(), - updated_at: new Date() - }) - .onConflict('messageId') - .ignore() - return undefined - } catch (error) { - if (hasErrorCode(error, 'ER_DUP_ENTRY')) { - return routeFailure(400, 'ERR_DUPLICATE_MESSAGE', 'Duplicate message.') - } - throw error - } + }) } async function notifyRecipient( @@ -513,29 +599,107 @@ async function storeMessages( payment: Payment | undefined, recipientOutputs: RecipientOutputs ): Promise>> { - const results: Array<{ recipient: string; messageId: string }> = [] - for (const recipient of validated.recipients) { - const messageId = validated.messageIdByRecipient.get(recipient) - if (messageId == null || messageId === '') { - return routeFailure( - 400, - 'ERR_INVALID_MESSAGEID', - `Missing messageId for recipient ${recipient}` + const resourceConfig = readMessageBoxResourceConfig() + const now = new Date() + const expiresAt = messageExpiresAt(resourceConfig, now) + + try { + const rows = await runtimeDeps.knex.transaction(async transaction => { + await acquireResourceLocks(transaction, [senderKey, ...validated.recipients], now) + + await transaction('messageBox') + .insert( + validated.recipients.map(identityKey => ({ + identityKey, + type: validated.boxType, + created_at: now, + updated_at: now + })) + ) + .onConflict(['type', 'identityKey']) + .ignore() + + const messageBoxes = await transaction('messageBox') + .whereIn('identityKey', validated.recipients) + .where('type', validated.boxType) + .select('identityKey', 'messageBoxId') + const messageBoxIds = new Map( + messageBoxes.map(row => [String(row.identityKey), Number(row.messageBoxId)]) ) - } - const failure = await storeMessage( - validated, - recipient, - messageId, - senderKey, - payment, - recipientOutputs + + const storedRows: StoredMessageRow[] = validated.recipients.map(recipient => { + const messageId = validated.messageIdByRecipient.get(recipient) + const messageBoxId = messageBoxIds.get(recipient) + if (messageId == null || messageId === '' || messageBoxId == null) { + throw new RouteFailureError( + routeFailure(400, 'ERR_INVALID_MESSAGEID', `Missing message data for ${recipient}`) + ) + } + const body = buildStoredBody(validated, recipient, payment, recipientOutputs) + return { + messageId, + messageBoxId, + sender: senderKey, + recipient, + body, + bodyBytes: Buffer.byteLength(body, 'utf8'), + created_at: now, + updated_at: now, + expires_at: expiresAt + } + }) + + const senderUsage = await resourceUsage(transaction, 'sender', senderKey, now) + enforceQuota( + senderUsage, + { + messageCount: storedRows.length, + bodyBytes: storedRows.reduce((total, row) => total + row.bodyBytes, 0) + }, + resourceConfig.maxSenderMessages, + resourceConfig.maxSenderBytes, + 'ERR_SENDER_QUOTA_EXCEEDED', + 'The sender storage quota has been reached. Retry after messages expire.' + ) + + for (const recipient of validated.recipients) { + const recipientRows = storedRows.filter(row => row.recipient === recipient) + const usage = await resourceUsage(transaction, 'recipient', recipient, now) + enforceQuota( + usage, + { + messageCount: recipientRows.length, + bodyBytes: recipientRows.reduce((total, row) => total + row.bodyBytes, 0) + }, + resourceConfig.maxInboxMessages, + resourceConfig.maxInboxBytes, + 'ERR_INBOX_QUOTA_EXCEEDED', + 'The recipient inbox storage quota has been reached. Retry after messages are acknowledged or expire.' + ) + } + + await transaction('messages').insert( + storedRows.map(({ bodyBytes: _bodyBytes, ...row }) => row) + ) + return storedRows + }) + + const results = rows.map(({ recipient, messageId }) => ({ recipient, messageId })) + await mapWithConcurrency( + results, + resourceConfig.notificationRecipientConcurrency, + async ({ recipient, messageId }) => { + await notifyRecipient(recipient, messageId, validated.boxType) + } ) - if (failure != null) return failure - results.push({ recipient, messageId }) - await notifyRecipient(recipient, messageId, validated.boxType) + return routeValue(results) + } catch (error) { + if (error instanceof RouteFailureError) return error.failure + if (isDuplicateDatabaseError(error)) { + return routeFailure(400, 'ERR_DUPLICATE_MESSAGE', 'Duplicate message.') + } + throw error } - return routeValue(results) } function sendFailure(res: Response, failure: RouteFailure): Response { @@ -667,8 +831,11 @@ export default { const validated = validateMessage(message) if (isRouteFailure(validated)) return sendFailure(res, validated) - await ensureMessageBoxes(validated.value.recipients, validated.value.boxType) - const deliveryFee = await getServerDeliveryFee(validated.value.boxType) + // BRC-105 pricing replaces the legacy server-delivery output. Recipient + // permission fees remain independent and are still honored. + const deliveryFee = readMessageBoxPricingConfig().enabled + ? 0 + : await getServerDeliveryFee(validated.value.boxType) const feeRows = await evaluateRecipientFees( validated.value.recipients, senderKey, diff --git a/infra/message-box-server/src/security/KnexPaymentReplayStore.ts b/infra/message-box-server/src/security/KnexPaymentReplayStore.ts new file mode 100644 index 000000000..0b0461da3 --- /dev/null +++ b/infra/message-box-server/src/security/KnexPaymentReplayStore.ts @@ -0,0 +1,50 @@ +import type { PaymentReplayStore } from '@bsv/payment-express-middleware' +import type { Knex } from 'knex' + +const DUPLICATE_CODES = new Set([ + 'ER_DUP_ENTRY', + 'SQLITE_CONSTRAINT_PRIMARYKEY', + 'SQLITE_CONSTRAINT_UNIQUE' +]) + +function isDuplicate(error: unknown): boolean { + if (error == null || typeof error !== 'object') return false + const { code, errno } = error as { code?: unknown; errno?: unknown } + return (typeof code === 'string' && DUPLICATE_CODES.has(code)) || errno === 1062 +} + +/** Durable, replica-safe BRC-105 transaction replay claims. */ +export class KnexPaymentReplayStore implements PaymentReplayStore { + constructor( + private readonly knex: Knex, + private readonly ttlDays: number = 365 + ) { + if (!Number.isSafeInteger(ttlDays) || (ttlDays !== -1 && ttlDays < 1)) { + throw new Error('Payment replay TTL must be -1 or a positive integer') + } + } + + async claim(transactionId: string): Promise { + const now = new Date() + const expiresAt = + this.ttlDays === -1 ? null : new Date(now.getTime() + this.ttlDays * 24 * 60 * 60 * 1_000) + try { + await this.knex('payment_replays').insert({ + transaction_id: transactionId, + created_at: now, + expires_at: expiresAt + }) + return true + } catch (error) { + if (isDuplicate(error)) return false + throw error + } + } + + async pruneExpired(now = new Date()): Promise { + return await this.knex('payment_replays') + .whereNotNull('expires_at') + .where('expires_at', '<=', now) + .delete() + } +} diff --git a/infra/message-box-server/src/security/edgePolicy.ts b/infra/message-box-server/src/security/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/infra/message-box-server/src/security/edgePolicy.ts +++ b/infra/message-box-server/src/security/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/infra/message-box-server/src/security/rateLimitPolicy.ts b/infra/message-box-server/src/security/rateLimitPolicy.ts index dea60bac5..87aa8b78e 100644 --- a/infra/message-box-server/src/security/rateLimitPolicy.ts +++ b/infra/message-box-server/src/security/rateLimitPolicy.ts @@ -29,6 +29,16 @@ export function readBoundedInteger ( return parsed } +function readRateLimit (name: string, fallback: number): number { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const normalized = value.trim().toLowerCase() + // express-rate-limit has no disabled sentinel. A safe-integer ceiling is + // effectively unlimited while preserving a numeric value for its internals. + if (normalized === '-1' || normalized === 'unlimited') return Number.MAX_SAFE_INTEGER + return readBoundedInteger(name, fallback, MAX_RATE_LIMIT) +} + export function rateLimitOptions ( prefix: string, defaults: RateLimitDefaults, @@ -36,7 +46,7 @@ export function rateLimitOptions ( ): Partial { return { windowMs: readBoundedInteger(`${prefix}_WINDOW_MS`, defaults.windowMs, MAX_RATE_LIMIT_WINDOW_MS), - limit: readBoundedInteger(`${prefix}_MAX`, defaults.limit, MAX_RATE_LIMIT), + limit: readRateLimit(`${prefix}_MAX`, defaults.limit), standardHeaders: 'draft-8', legacyHeaders: false, handler: (_req: Request, res: Response) => { diff --git a/infra/message-box-server/src/security/resourceMaintenance.ts b/infra/message-box-server/src/security/resourceMaintenance.ts new file mode 100644 index 000000000..71300d5b4 --- /dev/null +++ b/infra/message-box-server/src/security/resourceMaintenance.ts @@ -0,0 +1,67 @@ +import type { Knex } from 'knex' +import { readResourceLimit } from './edgePolicy.js' +import { Logger } from '../utils/logger.js' + +export interface MessageBoxMaintenance { + run: () => Promise + stop: () => void +} + +async function deleteExpired( + knex: Knex, + table: string, + column: string, + now: Date | number, + batchSize: number +): Promise { + const query = knex(table).whereNotNull(column).where(column, '<=', now) + if (batchSize !== -1) query.limit(batchSize) + return await query.delete() +} + +/** + * Bounded expiry maintenance. Multiple replicas may run this safely because + * every delete predicate is idempotent and database-atomic; operators can run + * a singleton maintenance role later without changing the data model. + */ +export function startMessageBoxMaintenance(knex: Knex): MessageBoxMaintenance { + const intervalMs = readResourceLimit( + 'MESSAGE_BOX', + 'RETENTION_CLEANUP_INTERVAL_MS', + 15 * 60 * 1_000 + ) + const batchSize = readResourceLimit('MESSAGE_BOX', 'RETENTION_CLEANUP_BATCH_SIZE', 1_000) + let running = false + let stopped = false + + const run = async (): Promise => { + if (running || stopped) return + running = true + try { + const now = new Date() + const [messages, replays, sessions] = await Promise.all([ + deleteExpired(knex, 'messages', 'expires_at', now, batchSize), + deleteExpired(knex, 'payment_replays', 'expires_at', now, batchSize), + deleteExpired(knex, 'auth_sessions', 'expiresAt', Date.now(), batchSize) + ]) + if (messages + replays + sessions > 0) { + Logger.log('[MAINTENANCE] Removed expired rows', { messages, replays, sessions }) + } + } catch (error) { + Logger.error('[MAINTENANCE] Failed to remove expired rows:', error) + } finally { + running = false + } + } + + void run() + const timer = intervalMs === -1 ? undefined : setInterval(() => void run(), intervalMs) + timer?.unref() + return { + run, + stop: () => { + stopped = true + if (timer != null) clearInterval(timer) + } + } +} diff --git a/infra/message-box-server/src/utils/boundedConcurrency.test.ts b/infra/message-box-server/src/utils/boundedConcurrency.test.ts new file mode 100644 index 000000000..9f7668ba6 --- /dev/null +++ b/infra/message-box-server/src/utils/boundedConcurrency.test.ts @@ -0,0 +1,25 @@ +import { mapWithConcurrency } from './boundedConcurrency.js' + +describe('mapWithConcurrency', () => { + it('preserves order while bounding active work', async () => { + let active = 0 + let maximumActive = 0 + const results = await mapWithConcurrency([3, 1, 2, 4], 2, async value => { + active += 1 + maximumActive = Math.max(maximumActive, active) + await Promise.resolve() + active -= 1 + return value * 2 + }) + + expect(results).toEqual([6, 2, 4, 8]) + expect(maximumActive).toBe(2) + }) + + it('accepts the explicit unlimited opt-out and rejects invalid limits', async () => { + await expect(mapWithConcurrency([1, 2], -1, async value => value)).resolves.toEqual([1, 2]) + await expect(mapWithConcurrency([1], 0, async value => value)).rejects.toThrow( + 'concurrency must be -1 or a positive safe integer' + ) + }) +}) diff --git a/infra/message-box-server/src/utils/boundedConcurrency.ts b/infra/message-box-server/src/utils/boundedConcurrency.ts new file mode 100644 index 000000000..9ba827d0b --- /dev/null +++ b/infra/message-box-server/src/utils/boundedConcurrency.ts @@ -0,0 +1,30 @@ +/** + * Run asynchronous work with a fixed number of workers while preserving + * result order. `-1` is the explicit operator opt-out used by resource limits. + */ +export async function mapWithConcurrency( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise { + if (concurrency !== -1 && (!Number.isSafeInteger(concurrency) || concurrency < 1)) { + throw new RangeError('concurrency must be -1 or a positive safe integer') + } + if (items.length === 0) return [] + if (concurrency === -1) { + return await Promise.all(items.map(async (item, index) => await worker(item, index))) + } + + const results = Array.from({ length: items.length }, () => undefined as R) + let nextIndex = 0 + const runWorker = async (): Promise => { + while (nextIndex < items.length) { + const index = nextIndex++ + results[index] = await worker(items[index], index) + } + } + await Promise.all( + Array.from({ length: Math.min(concurrency, items.length) }, async () => await runWorker()) + ) + return results +} diff --git a/infra/message-box-server/src/utils/sendFCMNotification.ts b/infra/message-box-server/src/utils/sendFCMNotification.ts index b7dbedc4b..d279b8f18 100644 --- a/infra/message-box-server/src/utils/sendFCMNotification.ts +++ b/infra/message-box-server/src/utils/sendFCMNotification.ts @@ -2,6 +2,8 @@ import { getFirebaseMessaging } from '../config/firebase.js' import { Logger } from './logger.js' import { PubKeyHex } from '@bsv/sdk' import { runtimeDeps } from '../runtimeDeps.js' +import { readMessageBoxResourceConfig } from '../config/resources.js' +import { mapWithConcurrency } from './boundedConcurrency.js' /** * FCM Payload interface @@ -33,13 +35,18 @@ export async function sendFCMNotification( Logger.log('[DEBUG] Payload:', payload) // Look up all active FCM tokens for this recipient - const deviceRegistrations = await runtimeDeps + const deviceQuery = runtimeDeps .knex('device_registrations') .where({ identity_key: recipient, active: true }) .select('fcm_token', 'platform', 'device_id') + .orderBy('updated_at', 'desc') + const resources = readMessageBoxResourceConfig() + const maxNotificationDevices = resources.maxNotificationDevices + if (maxNotificationDevices !== -1) deviceQuery.limit(maxNotificationDevices) + const deviceRegistrations = await deviceQuery if (deviceRegistrations.length === 0) { Logger.log(`[DEBUG] No active FCM tokens found for recipient ${recipient}`) @@ -49,94 +56,101 @@ export async function sendFCMNotification( Logger.log(`[DEBUG] Found ${deviceRegistrations.length} active device(s) for ${recipient}`) // Send notification to all registered devices - const sendPromises = deviceRegistrations.map(async device => { - try { - Logger.log( - `[DEBUG] Sending to ${device.platform ?? 'unknown'} device: ${device.device_id ?? 'unknown'}` - ) - - const messaging = getFirebaseMessaging() - if (messaging == null) { - return { - success: false, - token: device.fcm_token, - error: 'Firebase Messaging not initialized (ENABLE_FIREBASE != true)' + const results = await mapWithConcurrency( + deviceRegistrations, + resources.fcmSendConcurrency, + async device => { + try { + Logger.log( + `[DEBUG] Sending to ${device.platform ?? 'unknown'} device: ${device.device_id ?? 'unknown'}` + ) + + const messaging = getFirebaseMessaging() + if (messaging == null) { + return { + success: false, + token: device.fcm_token, + error: 'Firebase Messaging not initialized (ENABLE_FIREBASE != true)' + } } - } - await messaging.send({ - token: device.fcm_token, - notification: { - title: payload.title, - body: payload.messageId - }, - // Android configuration for headless service - android: { - priority: 'high', - data: { - messageId: payload.messageId, - originator: payload.originator || 'unknown' - } - }, - // iOS configuration for mutable content and Notification Service Extension - apns: { - headers: { - 'apns-push-type': 'alert', // required for iOS 13+ - 'apns-priority': '10' // deliver immediately - // optional: 'apns-topic': '' // FCM fills this automatically + await messaging.send({ + token: device.fcm_token, + notification: { + title: payload.title, + body: payload.messageId + }, + // Android configuration for headless service + android: { + priority: 'high', + data: { + messageId: payload.messageId, + originator: payload.originator || 'unknown' + } }, - payload: { - aps: { - 'mutable-content': 1, - alert: { - // include an alert so NSE can modify it - title: payload.title, - body: payload.messageId - } - // do NOT set 'content-available': 1 unless you also want background fetch + // iOS configuration for mutable content and Notification Service Extension + apns: { + headers: { + 'apns-push-type': 'alert', // required for iOS 13+ + 'apns-priority': '10' // deliver immediately + // optional: 'apns-topic': '' // FCM fills this automatically }, - // custom keys your NSE can read: - messageId: payload.messageId, - originator: payload.originator ?? 'unknown' + payload: { + aps: { + 'mutable-content': 1, + alert: { + // include an alert so NSE can modify it + title: payload.title, + body: payload.messageId + } + // do NOT set 'content-available': 1 unless you also want background fetch + }, + // custom keys your NSE can read: + messageId: payload.messageId, + originator: payload.originator ?? 'unknown' + } } - } - }) - - // Update last_used timestamp on successful send - await runtimeDeps.knex('device_registrations').where('fcm_token', device.fcm_token).update({ - last_used: new Date(), - updated_at: new Date() - }) + }) - return { success: true, token: device.fcm_token } - } catch (error) { - Logger.error(`[FCM ERROR] Failed to send to token ${device.fcm_token.slice(-10)}:`, error) - - // Mark token as inactive if it's invalid - if ( - error instanceof Error && - (error.message.includes('registration-token-not-registered') || - error.message.includes('invalid-registration-token')) - ) { - Logger.log(`[DEBUG] Marking invalid token as inactive: ...${device.fcm_token.slice(-10)}`) + // Update last_used timestamp on successful send await runtimeDeps .knex('device_registrations') .where('fcm_token', device.fcm_token) .update({ - active: false, + last_used: new Date(), updated_at: new Date() }) - } - return { - success: false, - token: device.fcm_token, - error: error instanceof Error ? error.message : String(error) + return { success: true, token: device.fcm_token } + } catch (error) { + Logger.error(`[FCM ERROR] Failed to send to token ${device.fcm_token.slice(-10)}:`, error) + + // Mark token as inactive if it's invalid + if ( + error instanceof Error && + (error.message.includes('registration-token-not-registered') || + error.message.includes('invalid-registration-token')) + ) { + Logger.log( + `[DEBUG] Marking invalid token as inactive: ...${device.fcm_token.slice(-10)}` + ) + await runtimeDeps + .knex('device_registrations') + .where('fcm_token', device.fcm_token) + .update({ + active: false, + updated_at: new Date() + }) + } + + return { + success: false, + token: device.fcm_token, + error: error instanceof Error ? error.message : String(error) + } } } - }) - - const results = await Promise.all(sendPromises) + ) const successCount = results.filter(r => r.success).length const failureCount = results.length - successCount diff --git a/infra/overlay-server/.env.example b/infra/overlay-server/.env.example index 0ce08f1f4..d8507ef7f 100644 --- a/infra/overlay-server/.env.example +++ b/infra/overlay-server/.env.example @@ -44,3 +44,23 @@ LOG_LEVEL=info # OVERLAY_FRAME_OPTIONS=disabled # OVERLAY_PERMISSIONS_POLICY=disabled # OVERLAY_STRICT_TRANSPORT_SECURITY=false + +# Bounded lookup, maintenance, admin, request, response, and connection policy. +OVERLAY_RESOURCE_PROFILE=standard +OVERLAY_MAX_BODY_BYTES=8388608 +OVERLAY_MAX_RESPONSE_BYTES=8388608 +OVERLAY_MAX_CONCURRENT_REQUESTS=24 +OVERLAY_MAX_CONNECTIONS=1000 +OVERLAY_MAX_LOOKUP_RESULTS=1000 +OVERLAY_MAX_BASM_TXIDS=1000 +OVERLAY_MAX_BASM_ANCHOR_RANGE=1000 +OVERLAY_ADMIN_LIST_DEFAULT_LIMIT=50 +OVERLAY_ADMIN_LIST_MAX_LIMIT=200 +OVERLAY_ADMIN_LIST_MAX_OFFSET=100000 +OVERLAY_JANITOR_BATCH_SIZE=250 +OVERLAY_JANITOR_MAX_REPORT_RESULTS=1000 +OVERLAY_REQUEST_TIMEOUT_MS=120000 +OVERLAY_HEADERS_TIMEOUT_MS=15000 +OVERLAY_KEEP_ALIVE_TIMEOUT_MS=5000 +OVERLAY_SOCKET_TIMEOUT_MS=120000 +OVERLAY_MAX_REQUESTS_PER_SOCKET=1000 diff --git a/infra/overlay-server/README.md b/infra/overlay-server/README.md index e6940d582..8c0eb1505 100644 --- a/infra/overlay-server/README.md +++ b/infra/overlay-server/README.md @@ -4,6 +4,9 @@ A set of ready-to-run configuration examples for stand-alone Overlay nodes built with [`@bsv/overlay-express`](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express). Use these examples to spin-up your own overlay infrastructure for distributed applications on Bitcoin SV. +Resource profiles and the custom-lookup safety contract are documented in +[Service Resource Profiles](../../docs/reference/service-resource-profiles.md). + --- ## Table of Contents diff --git a/infra/uhrp-server-basic/.env.example b/infra/uhrp-server-basic/.env.example index f4a2b566b..482fab6b3 100644 --- a/infra/uhrp-server-basic/.env.example +++ b/infra/uhrp-server-basic/.env.example @@ -27,9 +27,18 @@ LOG_LEVEL=info # UHRP_STRICT_TRANSPORT_SECURITY=false # Bounded request and connection policy. +UHRP_RESOURCE_PROFILE=standard UHRP_UPLOAD_MAX_BODY_BYTES=67108864 UHRP_JSON_MAX_BODY_BYTES=262144 -UHRP_MAX_CONCURRENT_REQUESTS=100 +UHRP_MAX_RESPONSE_BYTES=4194304 +UHRP_MAX_CONCURRENT_REQUESTS=64 +UHRP_MAX_CONNECTIONS=1000 +UHRP_LIST_DEFAULT_LIMIT=200 +UHRP_LIST_MAX_LIMIT=1000 +UHRP_LIST_MAX_OFFSET=100000 +UHRP_MAX_FILE_BYTES=11000000000 +UHRP_MAX_RETENTION_MINUTES=525600 +UHRP_MIME_CACHE_MAX_ENTRIES=10000 UHRP_REQUEST_TIMEOUT_MS=300000 UHRP_HEADERS_TIMEOUT_MS=15000 UHRP_KEEP_ALIVE_TIMEOUT_MS=5000 diff --git a/infra/uhrp-server-basic/README.md b/infra/uhrp-server-basic/README.md index 7de21bd82..ea2237412 100644 --- a/infra/uhrp-server-basic/README.md +++ b/infra/uhrp-server-basic/README.md @@ -2,6 +2,9 @@ For simple folk +See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) +for list, upload, retention, cache, response, and connection ceilings. + ## Request limits and trusted proxies Post-authentication routes use two rate-limit stages: 300 requests per minute diff --git a/infra/uhrp-server-basic/src/index.ts b/infra/uhrp-server-basic/src/index.ts index e2bd7e5d3..9de5ecf02 100644 --- a/infra/uhrp-server-basic/src/index.ts +++ b/infra/uhrp-server-basic/src/index.ts @@ -23,7 +23,12 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders } from './security/edgePolicy' import { createServiceHealth } from './serviceHealth' @@ -50,15 +55,21 @@ const authenticatedRateLimit = rateLimit(rateLimitOptions( )) const app = express() +const resourceProfile = readResourceProfile('UHRP') const serviceHealth = createServiceHealth() app.disable('x-powered-by') configureTrustProxy(app) +app.use(initialDoubleSlashCompatibility) app.use(securityHeaders({ environmentPrefix: 'UHRP' })) app.use(corsPolicy({ environmentPrefix: 'UHRP', methods: ['GET', 'PUT', 'POST', 'OPTIONS'] })) -app.use(concurrencyLimit('UHRP', 100)) +app.use(concurrencyLimit('UHRP', profileValue(resourceProfile, { + small: 16, + standard: 64, + highThroughput: 250 +}))) serviceHealth.register(app) app.use(preAuthRateLimit) // Add CDN MIME type middleware before static middleware @@ -69,6 +80,16 @@ app.use(bodyparser.json({ type: 'application/json' })) app.use(bodyParserErrorHandler) +const maxResponseBytes = readResourceLimit( + 'UHRP', + 'MAX_RESPONSE_BYTES', + profileValue(resourceProfile, { + small: 1024 * 1024, + standard: 4 * 1024 * 1024, + highThroughput: 16 * 1024 * 1024 + }) +) +app.use(responseSizeLimit('UHRP', maxResponseBytes)) app.use((req: Request, res: Response, next: NextFunction) => { log.info({ operation: 'request.in', method: req.method, path: req.path }, 'Incoming request') @@ -152,6 +173,9 @@ preAuthRoutes.filter(route => !(route as any).unsecured).forEach((route) => { }) app.use(authMiddleware); + // Re-apply after auth response interception so signed responses remain + // bounded with every compatible auth-middleware release. + app.use(responseSizeLimit('UHRP', maxResponseBytes)) app.use(authenticatedRateLimit) app.use(paymentMiddleware) diff --git a/infra/uhrp-server-basic/src/resourceLimits.ts b/infra/uhrp-server-basic/src/resourceLimits.ts new file mode 100644 index 000000000..8a1860684 --- /dev/null +++ b/infra/uhrp-server-basic/src/resourceLimits.ts @@ -0,0 +1,40 @@ +import { profileValue, readResourceLimit, readResourceProfile } from './security/edgePolicy' + +export function normalizeUhrpPagination( + limitValue: unknown, + offsetValue: unknown +): { limit: number; offset: number } { + const profile = readResourceProfile('UHRP') + const defaultLimit = readResourceLimit( + 'UHRP', + 'LIST_DEFAULT_LIMIT', + profileValue(profile, { small: 100, standard: 200, highThroughput: 1_000 }) + ) + const maxLimit = readResourceLimit( + 'UHRP', + 'LIST_MAX_LIMIT', + profileValue(profile, { small: 500, standard: 1_000, highThroughput: 5_000 }) + ) + const maxOffset = readResourceLimit( + 'UHRP', + 'LIST_MAX_OFFSET', + profileValue(profile, { small: 25_000, standard: 100_000, highThroughput: 1_000_000 }) + ) + if (defaultLimit !== -1 && maxLimit !== -1 && defaultLimit > maxLimit) { + throw new Error('UHRP_LIST_DEFAULT_LIMIT must not exceed UHRP_LIST_MAX_LIMIT') + } + let limit = Number(limitValue) + if (limitValue == null) { + limit = defaultLimit === -1 ? Number.MAX_SAFE_INTEGER : defaultLimit + } + const offset = offsetValue == null ? 0 : Number(offsetValue) + if (!Number.isSafeInteger(limit) || limit < 1 || (maxLimit !== -1 && limit > maxLimit)) { + const maximum = maxLimit === -1 ? '' : ` no greater than ${maxLimit}` + throw new RangeError(`limit must be a positive integer${maximum}`) + } + if (!Number.isSafeInteger(offset) || offset < 0 || (maxOffset !== -1 && offset > maxOffset)) { + const maximum = maxOffset === -1 ? '' : ` no greater than ${maxOffset}` + throw new RangeError(`offset must be a non-negative integer${maximum}`) + } + return { limit, offset } +} diff --git a/infra/uhrp-server-basic/src/routes/find.ts b/infra/uhrp-server-basic/src/routes/find.ts index faf374f70..c997302ca 100644 --- a/infra/uhrp-server-basic/src/routes/find.ts +++ b/infra/uhrp-server-basic/src/routes/find.ts @@ -1,6 +1,7 @@ import { Request, Response } from 'express' import { getMetadata } from '../utils/getMetadata' import { log } from '../logger' +import { normalizeUhrpPagination } from '../resourceLimits' interface FindRequest extends Request { auth: { @@ -8,6 +9,8 @@ interface FindRequest extends Request { } query: { uhrpUrl?: string + limit?: string + offset?: string } body: { limit?: number @@ -46,7 +49,10 @@ const findHandler = async (req: FindRequest, res: Response) => { } const { uhrpUrl } = req.query - const { limit, offset } = req.body + const pagination = normalizeUhrpPagination( + req.body?.limit ?? req.query.limit, + req.body?.offset ?? req.query.offset + ) if (!uhrpUrl) { return res.status(400).json({ status: 'error', @@ -60,7 +66,7 @@ const findHandler = async (req: FindRequest, res: Response) => { size, contentType, expiryTime - } = await getMetadata(uhrpUrl, identityKey, limit, offset) + } = await getMetadata(uhrpUrl, identityKey, pagination.limit, pagination.offset) return res.status(200).json({ status: 'success', @@ -72,6 +78,9 @@ const findHandler = async (req: FindRequest, res: Response) => { } }) } catch (error) { + if (error instanceof RangeError) { + return res.status(400).json({ status: 'error', code: 'ERR_INVALID_PAGINATION', description: error.message }) + } log.error({ operation: 'find.handle', outcome: 'error', err: error }, 'Error retrieving file metadata') return res.status(500).json({ status: 'error', @@ -99,4 +108,4 @@ export default { }, errors: ['ERR_NO_UHRP_URL', 'ERR_NOT_FOUND', 'ERR_FIND'], func: findHandler -} \ No newline at end of file +} diff --git a/infra/uhrp-server-basic/src/routes/list.ts b/infra/uhrp-server-basic/src/routes/list.ts index 18c174848..a22ce69ed 100644 --- a/infra/uhrp-server-basic/src/routes/list.ts +++ b/infra/uhrp-server-basic/src/routes/list.ts @@ -2,6 +2,7 @@ import { Request, Response } from 'express' import { getWallet } from '../utils/walletSingleton' import { Utils } from '@bsv/sdk' import { log } from '../logger' +import { normalizeUhrpPagination } from '../resourceLimits' interface ListRequest extends Request { auth: { @@ -43,7 +44,10 @@ const listHandler = async (req: ListRequest, res: Response) => { const wallet = await getWallet() - const { limit = 200, offset = 0 } = req.body + const { limit, offset } = normalizeUhrpPagination( + req.body?.limit ?? req.query.limit, + req.body?.offset ?? req.query.offset + ) const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', @@ -84,6 +88,9 @@ const listHandler = async (req: ListRequest, res: Response) => { uploads: result }) } catch (error) { + if (error instanceof RangeError) { + return res.status(400).json({ status: 'error', code: 'ERR_INVALID_PAGINATION', description: error.message }) + } log.error({ operation: 'list.handle', outcome: 'error', err: error }, 'Error listing advertisements') return res.status(500).json({ status: 'error', diff --git a/infra/uhrp-server-basic/src/routes/quote.ts b/infra/uhrp-server-basic/src/routes/quote.ts index 5b8f9f361..e931756f2 100644 --- a/infra/uhrp-server-basic/src/routes/quote.ts +++ b/infra/uhrp-server-basic/src/routes/quote.ts @@ -1,6 +1,7 @@ import { Request, Response } from 'express' import getPriceForFile from '../utils/getPriceForFile' import { log } from '../logger' +import { readResourceLimit } from '../security/edgePolicy' const { MIN_HOSTING_MINUTES @@ -54,6 +55,14 @@ const quoteHandler = async (req: QuoteRequest, res: Response) => 'The file size must be an integer.' }) } + const maxFileBytes = readResourceLimit('UHRP', 'MAX_FILE_BYTES', 11_000_000_000) + if (maxFileBytes !== -1 && fileSize > maxFileBytes) { + return res.status(400).json({ + status: 'error', + code: 'ERR_INVALID_SIZE', + description: `The file size must not exceed ${maxFileBytes} bytes.` + }) + } const minHostingMinutes = Number(MIN_HOSTING_MINUTES) || 0 @@ -66,12 +75,12 @@ const quoteHandler = async (req: QuoteRequest, res: Response) => }) } - // Retention period must not be more than 69 million minutes - if (retentionPeriod > 69000000) { + const maxRetentionMinutes = readResourceLimit('UHRP', 'MAX_RETENTION_MINUTES', 525_600) + if (maxRetentionMinutes !== -1 && retentionPeriod > maxRetentionMinutes) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_RETENTION_PERIOD', - description: 'The retention period must be less than 69 million minutes (about 130 years)' + description: `The retention period must not exceed ${maxRetentionMinutes} minutes.` }) } @@ -108,4 +117,4 @@ export default { 'ERR_INTERNAL' ], func: quoteHandler -} \ No newline at end of file +} diff --git a/infra/uhrp-server-basic/src/routes/renew.ts b/infra/uhrp-server-basic/src/routes/renew.ts index 57910d025..cb743ff0f 100644 --- a/infra/uhrp-server-basic/src/routes/renew.ts +++ b/infra/uhrp-server-basic/src/routes/renew.ts @@ -4,6 +4,8 @@ import { getWallet } from '../utils/walletSingleton' import { PushDrop, SHIPBroadcaster, Transaction, Utils } from '@bsv/sdk' import { getMetadata } from '../utils/getMetadata' import { log } from '../logger' +import { normalizeUhrpPagination } from '../resourceLimits' +import { readResourceLimit } from '../security/edgePolicy' const BSV_NETWORK = process.env.BSV_NETWORK as 'mainnet' | 'testnet' @@ -93,18 +95,21 @@ const renewHandler = async (req: RenewRequest, res: Response) => description: 'Missing objectIdentifier or additionalMinutes.' }) } - if (additionalMinutes <= 0) { + const maxRetentionMinutes = readResourceLimit('UHRP', 'MAX_RETENTION_MINUTES', 525_600) + if (!Number.isSafeInteger(additionalMinutes) || additionalMinutes <= 0 || + (maxRetentionMinutes !== -1 && additionalMinutes > maxRetentionMinutes)) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_TIME', description: 'Additional Minutes must be a positive integer' }) } + const pagination = normalizeUhrpPagination(limit, offset) const { objectIdentifier, size, expiryTime: prevExpiryTime - } = await getMetadata(uhrpUrl, identityKey, limit, offset) + } = await getMetadata(uhrpUrl, identityKey, pagination.limit, pagination.offset) // Convert to MS to create an ISO string const newExpiryTimeSeconds = prevExpiryTime + (additionalMinutes * 60) @@ -119,8 +124,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => tagQueryMode: 'all', includeTags: true, include: 'entire transactions', - limit: limit ?? 200, - offset: offset ?? 0 + ...pagination }) if (!outputs || outputs.length === 0) { diff --git a/infra/uhrp-server-basic/src/routes/upload.ts b/infra/uhrp-server-basic/src/routes/upload.ts index f113798a5..d9b46c96a 100644 --- a/infra/uhrp-server-basic/src/routes/upload.ts +++ b/infra/uhrp-server-basic/src/routes/upload.ts @@ -4,6 +4,7 @@ import { Utils } from '@bsv/sdk' import getPriceForFile from '../utils/getPriceForFile' import getUploadURL from '../utils/getUploadURL' import { log } from '../logger' +import { readResourceLimit } from '../security/edgePolicy' const MIN_HOSTING_MINUTES = process.env.MIN_HOSTING_MINUTES @@ -45,7 +46,7 @@ export async function uploadHandler(req: UploadRequest, res: Response= ${minHostingMinutes} minutes` }) } + if (maxRetentionMinutes !== -1 && retentionPeriod > maxRetentionMinutes) { + return res.status(400).json({ + status: 'error', + code: 'ERR_INVALID_RETENTION_PERIOD', + description: `The retention period must not exceed ${maxRetentionMinutes} minutes.` + }) + } - if (fileSize > 11000000000) { + const maxFileBytes = readResourceLimit('UHRP', 'MAX_FILE_BYTES', 11_000_000_000) + if (maxFileBytes !== -1 && fileSize > maxFileBytes) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_SIZE', - description: 'Max supported file size is 11000000000 bytes.' + description: `Max supported file size is ${maxFileBytes} bytes.` }) } diff --git a/infra/uhrp-server-basic/src/security/edgePolicy.ts b/infra/uhrp-server-basic/src/security/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/infra/uhrp-server-basic/src/security/edgePolicy.ts +++ b/infra/uhrp-server-basic/src/security/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/infra/uhrp-server-basic/src/security/rateLimitPolicy.ts b/infra/uhrp-server-basic/src/security/rateLimitPolicy.ts index dea60bac5..87aa8b78e 100644 --- a/infra/uhrp-server-basic/src/security/rateLimitPolicy.ts +++ b/infra/uhrp-server-basic/src/security/rateLimitPolicy.ts @@ -29,6 +29,16 @@ export function readBoundedInteger ( return parsed } +function readRateLimit (name: string, fallback: number): number { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const normalized = value.trim().toLowerCase() + // express-rate-limit has no disabled sentinel. A safe-integer ceiling is + // effectively unlimited while preserving a numeric value for its internals. + if (normalized === '-1' || normalized === 'unlimited') return Number.MAX_SAFE_INTEGER + return readBoundedInteger(name, fallback, MAX_RATE_LIMIT) +} + export function rateLimitOptions ( prefix: string, defaults: RateLimitDefaults, @@ -36,7 +46,7 @@ export function rateLimitOptions ( ): Partial { return { windowMs: readBoundedInteger(`${prefix}_WINDOW_MS`, defaults.windowMs, MAX_RATE_LIMIT_WINDOW_MS), - limit: readBoundedInteger(`${prefix}_MAX`, defaults.limit, MAX_RATE_LIMIT), + limit: readRateLimit(`${prefix}_MAX`, defaults.limit), standardHeaders: 'draft-8', legacyHeaders: false, handler: (_req: Request, res: Response) => { diff --git a/infra/uhrp-server-basic/src/serviceHealth.ts b/infra/uhrp-server-basic/src/serviceHealth.ts index 89814fa3e..c0a4c30b1 100644 --- a/infra/uhrp-server-basic/src/serviceHealth.ts +++ b/infra/uhrp-server-basic/src/serviceHealth.ts @@ -20,6 +20,9 @@ export const createServiceHealth = (): ServiceHealth => { app.get('/health', (_req: Request, res: Response) => { res.status(200).json({ status: 'ok', live: true }) }) + app.get('/healthz', (_req: Request, res: Response) => { + res.status(200).json({ status: 'ok', live: true }) + }) app.get('/ready', (_req: Request, res: Response) => { res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'starting', ready }) }) diff --git a/infra/uhrp-server-basic/src/utils/getMetadata.ts b/infra/uhrp-server-basic/src/utils/getMetadata.ts index d4b52b8d1..59321f3e8 100644 --- a/infra/uhrp-server-basic/src/utils/getMetadata.ts +++ b/infra/uhrp-server-basic/src/utils/getMetadata.ts @@ -1,5 +1,6 @@ import { getWallet } from './walletSingleton' import { Utils } from '@bsv/sdk' +import { normalizeUhrpPagination } from '../resourceLimits' interface FileMetadata { @@ -20,13 +21,13 @@ interface FileMetadata { */ export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, limit?: number, offset?: number): Promise { const wallet = await getWallet() + const pagination = normalizeUhrpPagination(limit, offset) const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `uploader_identity_key_${uploaderIdentityKey}`], tagQueryMode: 'all', includeTags: true, - limit: limit ?? 200, - offset: offset ?? 0 + ...pagination }) let objectIdentifier, name, contentType, size diff --git a/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts b/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts index cedb8636f..097ed4cff 100644 --- a/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts +++ b/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts @@ -5,6 +5,7 @@ import { getWallet } from './walletSingleton' import { Utils } from '@bsv/sdk' import { log } from '../logger' import { CDN_ROOT } from './cdnObjectPath' +import { profileValue, readResourceLimit, readResourceProfile } from '../security/edgePolicy' /** * Cache to store MIME types for object identifiers to avoid repeated database lookups @@ -12,6 +13,15 @@ import { CDN_ROOT } from './cdnObjectPath' const mimeTypeCache = new Map() const CACHE_TTL = 5 * 60 * 1000 // 5 minutes in milliseconds const cacheTimestamps = new Map() +const MAX_MIME_CACHE_ENTRIES = readResourceLimit( + 'UHRP', + 'MIME_CACHE_MAX_ENTRIES', + profileValue(readResourceProfile('UHRP'), { + small: 2_500, + standard: 10_000, + highThroughput: 50_000 + }) +) const FILE_SIGNATURES = [ { bytes: [0xff, 0xd8, 0xff], mimeType: 'image/jpeg' }, { bytes: [0x89, 0x50, 0x4e, 0x47], mimeType: 'image/png' }, @@ -20,6 +30,35 @@ const FILE_SIGNATURES = [ { bytes: [0x50, 0x4b], mimeType: 'application/zip' } ] as const +function latestAdvertisedMimeType(outputs: Array<{ tags?: string[] }>): string | null { + let mimeType: string | null = null + let maxExpiry = 0 + for (const output of outputs) { + const contentTypeTag = output.tags?.find(tag => tag.startsWith('content_type_')) + const expiryTag = output.tags?.find(tag => tag.startsWith('expiry_time_')) + if (contentTypeTag == null || expiryTag == null) continue + + const expiryTime = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + if (expiryTime <= Date.now() / 1000 || expiryTime <= maxExpiry) continue + maxExpiry = expiryTime + mimeType = contentTypeTag.substring('content_type_'.length) + } + return mimeType +} + +function cacheMimeType(cacheKey: string, mimeType: string): void { + if (MAX_MIME_CACHE_ENTRIES !== -1) { + while (mimeTypeCache.size >= MAX_MIME_CACHE_ENTRIES) { + const oldest = mimeTypeCache.keys().next().value + if (oldest == null) break + mimeTypeCache.delete(oldest) + cacheTimestamps.delete(oldest) + } + } + mimeTypeCache.set(cacheKey, mimeType) + cacheTimestamps.set(cacheKey, Date.now()) +} + /** * Get MIME type from UHRP advertisement tags */ @@ -30,8 +69,17 @@ async function getMimeTypeFromAdvertisement(objectIdentifier: string): Promise t.startsWith('content_type_')) - const expiryTag = output.tags.find(t => t.startsWith('expiry_time_')) - - if (contentTypeTag && expiryTag) { - const expiryTime = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 - - // Only consider non-expired advertisements - if (expiryTime > Date.now() / 1000 && expiryTime > maxExpiry) { - maxExpiry = expiryTime - mimeType = contentTypeTag.substring('content_type_'.length) - } - } - } + const mimeType = latestAdvertisedMimeType(outputs) // Cache the result (even if null) - if (mimeType) { - mimeTypeCache.set(cacheKey, mimeType) - cacheTimestamps.set(cacheKey, Date.now()) - } + if (mimeType != null) cacheMimeType(cacheKey, mimeType) return mimeType } catch (error) { @@ -99,12 +125,16 @@ function isJson(text: string): boolean { } function detectMimeTypeFromContent(filePath: string): string { + let descriptor: number | undefined try { - const buffer = fs.readFileSync(filePath, { encoding: null }) - const binaryMimeType = detectBinaryMimeType(buffer) + descriptor = fs.openSync(filePath, 'r') + const buffer = Buffer.alloc(512) + const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, 0) + const sample = buffer.subarray(0, bytesRead) + const binaryMimeType = detectBinaryMimeType(sample) if (binaryMimeType != null) return binaryMimeType - const textSample = buffer.slice(0, 512).toString('utf8', 0, Math.min(512, buffer.length)) + const textSample = sample.toString('utf8') if (!/^[\x21-\x7E\s]*$/.test(textSample)) return 'application/octet-stream' const trimmedSample = textSample.trim() @@ -117,6 +147,8 @@ function detectMimeTypeFromContent(filePath: string): string { return 'text/plain' } catch { return 'application/octet-stream' + } finally { + if (descriptor != null) fs.closeSync(descriptor) } } diff --git a/infra/uhrp-server-cloud-bucket/README.md b/infra/uhrp-server-cloud-bucket/README.md index e6790c895..619e1d14f 100644 --- a/infra/uhrp-server-cloud-bucket/README.md +++ b/infra/uhrp-server-cloud-bucket/README.md @@ -1,5 +1,8 @@ # UHRP Storage Server – Deployment Guide +See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) +for list, retention, response, connection, and provider-scaling guidance. + This guide walks you through deploying **UHRP Storage Server** on Google Cloud Platform (GCP) with continuous delivery via GitHub Actions. When you finish, you’ll have: - A single‑region **Cloud Storage bucket** that stores all UHRP data. diff --git a/infra/uhrp-server-cloud-bucket/secrets/.env.example b/infra/uhrp-server-cloud-bucket/secrets/.env.example index 766e2e4d4..a3b63507f 100644 --- a/infra/uhrp-server-cloud-bucket/secrets/.env.example +++ b/infra/uhrp-server-cloud-bucket/secrets/.env.example @@ -34,8 +34,16 @@ LOG_LEVEL=info # UHRP_STRICT_TRANSPORT_SECURITY=false # Bounded request and connection policy. +UHRP_RESOURCE_PROFILE=standard UHRP_JSON_MAX_BODY_BYTES=262144 -UHRP_MAX_CONCURRENT_REQUESTS=200 +UHRP_MAX_RESPONSE_BYTES=4194304 +UHRP_MAX_CONCURRENT_REQUESTS=64 +UHRP_MAX_CONNECTIONS=1000 +UHRP_LIST_DEFAULT_LIMIT=200 +UHRP_LIST_MAX_LIMIT=1000 +UHRP_LIST_MAX_OFFSET=100000 +UHRP_MAX_FILE_BYTES=11000000000 +UHRP_MAX_RETENTION_MINUTES=525600 UHRP_REQUEST_TIMEOUT_MS=60000 UHRP_HEADERS_TIMEOUT_MS=15000 UHRP_KEEP_ALIVE_TIMEOUT_MS=5000 diff --git a/infra/uhrp-server-cloud-bucket/src/index.ts b/infra/uhrp-server-cloud-bucket/src/index.ts index 282f8ecb7..d7b651474 100644 --- a/infra/uhrp-server-cloud-bucket/src/index.ts +++ b/infra/uhrp-server-cloud-bucket/src/index.ts @@ -20,7 +20,12 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders } from './security/edgePolicy' import { createServiceHealth } from './serviceHealth' @@ -42,15 +47,21 @@ const authenticatedRateLimit = rateLimit(rateLimitOptions( )) const app = express() +const resourceProfile = readResourceProfile('UHRP') const serviceHealth = createServiceHealth() app.disable('x-powered-by') configureTrustProxy(app) +app.use(initialDoubleSlashCompatibility) app.use(securityHeaders({ environmentPrefix: 'UHRP' })) app.use(corsPolicy({ environmentPrefix: 'UHRP', methods: ['GET', 'POST', 'OPTIONS'] })) -app.use(concurrencyLimit('UHRP', 200)) +app.use(concurrencyLimit('UHRP', profileValue(resourceProfile, { + small: 16, + standard: 64, + highThroughput: 250 +}))) serviceHealth.register(app) app.use(preAuthRateLimit) app.use(bodyparser.json({ @@ -58,6 +69,16 @@ app.use(bodyparser.json({ type: 'application/json' })) app.use(bodyParserErrorHandler) +const maxResponseBytes = readResourceLimit( + 'UHRP', + 'MAX_RESPONSE_BYTES', + profileValue(resourceProfile, { + small: 1024 * 1024, + standard: 4 * 1024 * 1024, + highThroughput: 16 * 1024 * 1024 + }) +) +app.use(responseSizeLimit('UHRP', maxResponseBytes)) app.use((req: Request, res: Response, next: NextFunction) => { log.info({ operation: 'request.in', method: req.method, url: req.url }, 'Incoming request') @@ -156,6 +177,9 @@ preAuthRoutes.filter(route => !(route as any).unsecured).forEach((route) => { }) app.use(authMiddleware); + // Re-apply after auth response interception so signed responses remain + // bounded with every compatible auth-middleware release. + app.use(responseSizeLimit('UHRP', maxResponseBytes)) app.use(authenticatedRateLimit) app.use(paymentMiddleware) diff --git a/infra/uhrp-server-cloud-bucket/src/resourceLimits.ts b/infra/uhrp-server-cloud-bucket/src/resourceLimits.ts new file mode 100644 index 000000000..8a1860684 --- /dev/null +++ b/infra/uhrp-server-cloud-bucket/src/resourceLimits.ts @@ -0,0 +1,40 @@ +import { profileValue, readResourceLimit, readResourceProfile } from './security/edgePolicy' + +export function normalizeUhrpPagination( + limitValue: unknown, + offsetValue: unknown +): { limit: number; offset: number } { + const profile = readResourceProfile('UHRP') + const defaultLimit = readResourceLimit( + 'UHRP', + 'LIST_DEFAULT_LIMIT', + profileValue(profile, { small: 100, standard: 200, highThroughput: 1_000 }) + ) + const maxLimit = readResourceLimit( + 'UHRP', + 'LIST_MAX_LIMIT', + profileValue(profile, { small: 500, standard: 1_000, highThroughput: 5_000 }) + ) + const maxOffset = readResourceLimit( + 'UHRP', + 'LIST_MAX_OFFSET', + profileValue(profile, { small: 25_000, standard: 100_000, highThroughput: 1_000_000 }) + ) + if (defaultLimit !== -1 && maxLimit !== -1 && defaultLimit > maxLimit) { + throw new Error('UHRP_LIST_DEFAULT_LIMIT must not exceed UHRP_LIST_MAX_LIMIT') + } + let limit = Number(limitValue) + if (limitValue == null) { + limit = defaultLimit === -1 ? Number.MAX_SAFE_INTEGER : defaultLimit + } + const offset = offsetValue == null ? 0 : Number(offsetValue) + if (!Number.isSafeInteger(limit) || limit < 1 || (maxLimit !== -1 && limit > maxLimit)) { + const maximum = maxLimit === -1 ? '' : ` no greater than ${maxLimit}` + throw new RangeError(`limit must be a positive integer${maximum}`) + } + if (!Number.isSafeInteger(offset) || offset < 0 || (maxOffset !== -1 && offset > maxOffset)) { + const maximum = maxOffset === -1 ? '' : ` no greater than ${maxOffset}` + throw new RangeError(`offset must be a non-negative integer${maximum}`) + } + return { limit, offset } +} diff --git a/infra/uhrp-server-cloud-bucket/src/routes/find.ts b/infra/uhrp-server-cloud-bucket/src/routes/find.ts index b42abbe67..d1ba5d6b7 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/find.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/find.ts @@ -1,6 +1,7 @@ import { Request, Response } from 'express' import { getMetadata } from '../utils/getMetadata' import { log } from '../logger' +import { normalizeUhrpPagination } from '../resourceLimits' interface FindRequest extends Request { auth: { @@ -8,6 +9,8 @@ interface FindRequest extends Request { } query: { uhrpUrl?: string + limit?: string + offset?: string } body: { limit?: number @@ -39,7 +42,10 @@ const findHandler = async (req: FindRequest, res: Response) => { } const { uhrpUrl } = req.query - const { limit, offset } = req.body + const pagination = normalizeUhrpPagination( + req.body?.limit ?? req.query.limit, + req.body?.offset ?? req.query.offset + ) if (!uhrpUrl) { return res.status(400).json({ status: 'error', @@ -53,7 +59,7 @@ const findHandler = async (req: FindRequest, res: Response) => { size, contentType, expiryTime - } = await getMetadata(uhrpUrl, identityKey, limit, offset) + } = await getMetadata(uhrpUrl, identityKey, pagination.limit, pagination.offset) return res.status(200).json({ status: 'success', @@ -65,6 +71,9 @@ const findHandler = async (req: FindRequest, res: Response) => { } }) } catch (error) { + if (error instanceof RangeError) { + return res.status(400).json({ status: 'error', code: 'ERR_INVALID_PAGINATION', description: error.message }) + } log.error({ operation: 'find.handle', outcome: 'error', err: error }, 'Find handler failed') return res.status(500).json({ status: 'error', diff --git a/infra/uhrp-server-cloud-bucket/src/routes/list.ts b/infra/uhrp-server-cloud-bucket/src/routes/list.ts index 379d43512..f22f592db 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/list.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/list.ts @@ -2,6 +2,7 @@ import { Request, Response } from 'express' import { getWallet } from '../utils/walletSingleton' import { Utils } from '@bsv/sdk' import { log } from '../logger' +import { normalizeUhrpPagination } from '../resourceLimits' interface ListRequest extends Request { auth: { @@ -36,7 +37,10 @@ const listHandler = async (req: ListRequest, res: Response) => { const wallet = await getWallet() - const { limit = 200, offset = 0 } = req.body + const { limit, offset } = normalizeUhrpPagination( + req.body?.limit ?? req.query.limit, + req.body?.offset ?? req.query.offset + ) const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', @@ -77,6 +81,9 @@ const listHandler = async (req: ListRequest, res: Response) => { uploads: result }) } catch (error) { + if (error instanceof RangeError) { + return res.status(400).json({ status: 'error', code: 'ERR_INVALID_PAGINATION', description: error.message }) + } log.error({ operation: 'list.handle', outcome: 'error', err: error }, 'List handler failed') return res.status(500).json({ status: 'error', diff --git a/infra/uhrp-server-cloud-bucket/src/routes/quote.ts b/infra/uhrp-server-cloud-bucket/src/routes/quote.ts index 7743335c6..5f9eb64f1 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/quote.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/quote.ts @@ -1,6 +1,7 @@ import { Request, Response } from 'express' import getPriceForFile from '../utils/getPriceForFile' import { log } from '../logger' +import { readResourceLimit } from '../security/edgePolicy' const { MIN_HOSTING_MINUTES @@ -54,6 +55,14 @@ const quoteHandler = async (req: QuoteRequest, res: Response) => 'The file size must be an integer.' }) } + const maxFileBytes = readResourceLimit('UHRP', 'MAX_FILE_BYTES', 11_000_000_000) + if (maxFileBytes !== -1 && fileSize > maxFileBytes) { + return res.status(400).json({ + status: 'error', + code: 'ERR_INVALID_SIZE', + description: `The file size must not exceed ${maxFileBytes} bytes.` + }) + } const minHostingMinutes = Number(MIN_HOSTING_MINUTES) || 0 @@ -66,12 +75,12 @@ const quoteHandler = async (req: QuoteRequest, res: Response) => }) } - // Retention period must not be more than 69 million minutes - if (retentionPeriod > 69000000) { + const maxRetentionMinutes = readResourceLimit('UHRP', 'MAX_RETENTION_MINUTES', 525_600) + if (maxRetentionMinutes !== -1 && retentionPeriod > maxRetentionMinutes) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_RETENTION_PERIOD', - description: 'The retention period must be less than 69 million minutes (about 130 years)' + description: `The retention period must not exceed ${maxRetentionMinutes} minutes.` }) } @@ -108,4 +117,4 @@ export default { 'ERR_INTERNAL' ], func: quoteHandler -} \ No newline at end of file +} diff --git a/infra/uhrp-server-cloud-bucket/src/routes/renew.ts b/infra/uhrp-server-cloud-bucket/src/routes/renew.ts index 7a99f6cb7..5b1299e64 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/renew.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/renew.ts @@ -5,6 +5,8 @@ import { getWallet } from '../utils/walletSingleton' import { PushDrop, SHIPBroadcaster, Transaction, Utils } from '@bsv/sdk' import { getMetadata } from '../utils/getMetadata' import { log } from '../logger' +import { normalizeUhrpPagination } from '../resourceLimits' +import { readResourceLimit } from '../security/edgePolicy' const storage = new Storage() const GCP_BUCKET_NAME = process.env.GCP_BUCKET_NAME as string @@ -97,18 +99,21 @@ const renewHandler = async (req: RenewRequest, res: Response) => description: 'Missing objectIdentifier or additionalMinutes.' }) } - if (additionalMinutes <= 0) { + const maxRetentionMinutes = readResourceLimit('UHRP', 'MAX_RETENTION_MINUTES', 525_600) + if (!Number.isSafeInteger(additionalMinutes) || additionalMinutes <= 0 || + (maxRetentionMinutes !== -1 && additionalMinutes > maxRetentionMinutes)) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_TIME', description: 'Additional Minutes must be a positive integer' }) } + const pagination = normalizeUhrpPagination(limit, offset) const { objectIdentifier, size, expiryTime: prevExpiryTime - } = await getMetadata(uhrpUrl, identityKey, limit, offset) + } = await getMetadata(uhrpUrl, identityKey, pagination.limit, pagination.offset) // Convert to MS to create an ISO string const newExpiryTimeSeconds = prevExpiryTime + (additionalMinutes * 60) @@ -124,8 +129,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => tagQueryMode: 'all', includeTags: true, include: 'entire transactions', - limit: limit ?? 200, - offset: offset ?? 0 + ...pagination }) if (!outputs || outputs.length === 0) { diff --git a/infra/uhrp-server-cloud-bucket/src/routes/upload.ts b/infra/uhrp-server-cloud-bucket/src/routes/upload.ts index 8971ee7ef..22a18c6b4 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/upload.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/upload.ts @@ -4,6 +4,7 @@ import { Utils } from '@bsv/sdk' import getPriceForFile from '../utils/getPriceForFile' import getUploadURL from '../utils/getUploadURL' import { log } from '../logger' +import { readResourceLimit } from '../security/edgePolicy' const MIN_HOSTING_MINUTES = process.env.MIN_HOSTING_MINUTES @@ -45,7 +46,7 @@ export async function uploadHandler(req: UploadRequest, res: Response= ${minHostingMinutes} minutes` }) } + if (maxRetentionMinutes !== -1 && retentionPeriod > maxRetentionMinutes) { + return res.status(400).json({ + status: 'error', + code: 'ERR_INVALID_RETENTION_PERIOD', + description: `The retention period must not exceed ${maxRetentionMinutes} minutes.` + }) + } - if (fileSize > 11000000000) { + const maxFileBytes = readResourceLimit('UHRP', 'MAX_FILE_BYTES', 11_000_000_000) + if (maxFileBytes !== -1 && fileSize > maxFileBytes) { return res.status(400).json({ status: 'error', code: 'ERR_INVALID_SIZE', - description: 'Max supported file size is 11000000000 bytes.' + description: `Max supported file size is ${maxFileBytes} bytes.` }) } diff --git a/infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts b/infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts +++ b/infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts b/infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts index dea60bac5..87aa8b78e 100644 --- a/infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts +++ b/infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts @@ -29,6 +29,16 @@ export function readBoundedInteger ( return parsed } +function readRateLimit (name: string, fallback: number): number { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const normalized = value.trim().toLowerCase() + // express-rate-limit has no disabled sentinel. A safe-integer ceiling is + // effectively unlimited while preserving a numeric value for its internals. + if (normalized === '-1' || normalized === 'unlimited') return Number.MAX_SAFE_INTEGER + return readBoundedInteger(name, fallback, MAX_RATE_LIMIT) +} + export function rateLimitOptions ( prefix: string, defaults: RateLimitDefaults, @@ -36,7 +46,7 @@ export function rateLimitOptions ( ): Partial { return { windowMs: readBoundedInteger(`${prefix}_WINDOW_MS`, defaults.windowMs, MAX_RATE_LIMIT_WINDOW_MS), - limit: readBoundedInteger(`${prefix}_MAX`, defaults.limit, MAX_RATE_LIMIT), + limit: readRateLimit(`${prefix}_MAX`, defaults.limit), standardHeaders: 'draft-8', legacyHeaders: false, handler: (_req: Request, res: Response) => { diff --git a/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts b/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts index 4f4b81de8..d05c31031 100644 --- a/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts +++ b/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts @@ -19,6 +19,7 @@ class CloudServiceHealth implements ServiceHealth { public readonly register = (app: Express): void => { app.get('/health', this.reportLiveness) + app.get('/healthz', this.reportLiveness) app.get('/ready', this.reportReadiness) } diff --git a/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts b/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts index e5ece0084..9887a2d1b 100644 --- a/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts +++ b/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts @@ -2,6 +2,7 @@ import { Storage } from '@google-cloud/storage' import { getWallet } from './walletSingleton' import { Utils } from '@bsv/sdk' +import { normalizeUhrpPagination } from '../resourceLimits' const storage = new Storage() const { GCP_BUCKET_NAME } = process.env @@ -24,13 +25,13 @@ interface FileMetadata { */ export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, limit?: number, offset?: number): Promise { const wallet = await getWallet() + const pagination = normalizeUhrpPagination(limit, offset) const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `uploader_identity_key_${uploaderIdentityKey}`], tagQueryMode: 'all', includeTags: true, - limit: limit ?? 200, - offset: offset ?? 0 + ...pagination }) let objectIdentifier diff --git a/infra/wab/.env.example b/infra/wab/.env.example index b827eaf6d..8a1da5b36 100644 --- a/infra/wab/.env.example +++ b/infra/wab/.env.example @@ -43,13 +43,18 @@ LOG_LEVEL=info # WAB_STRICT_TRANSPORT_SECURITY=false # Bounded request, connection, and rate-limit policy +WAB_RESOURCE_PROFILE=standard WAB_MAX_BODY_BYTES=262144 -WAB_MAX_CONCURRENT_REQUESTS=200 +WAB_MAX_RESPONSE_BYTES=2097152 +WAB_MAX_CONCURRENT_REQUESTS=128 +WAB_MAX_CONNECTIONS=1000 WAB_REQUEST_TIMEOUT_MS=30000 WAB_HEADERS_TIMEOUT_MS=10000 WAB_KEEP_ALIVE_TIMEOUT_MS=5000 WAB_SOCKET_TIMEOUT_MS=30000 WAB_MAX_REQUESTS_PER_SOCKET=1000 +WAB_DB_POOL_MIN=2 +WAB_DB_POOL_MAX=10 WAB_PRE_AUTH_RATE_LIMIT_MAX=300 WAB_PRE_AUTH_RATE_LIMIT_WINDOW_MS=60000 WAB_AUTH_RATE_LIMIT_MAX=10 diff --git a/infra/wab/README.md b/infra/wab/README.md index 14ba1adbf..216968031 100644 --- a/infra/wab/README.md +++ b/infra/wab/README.md @@ -2,6 +2,9 @@ Welcome to the **Wallet Authentication Backend (WAB)** project! This README provides a **comprehensive, ground-up guide** to help you **understand**, **configure**, **deploy**, and **run** your own WAB server. +See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) +for bounded defaults, database sizing, and HPA prerequisites. + --- ## What Is the WAB? diff --git a/infra/wab/src/__tests/rateLimitPolicy.test.ts b/infra/wab/src/__tests/rateLimitPolicy.test.ts index 0e0a45cc7..41a601132 100644 --- a/infra/wab/src/__tests/rateLimitPolicy.test.ts +++ b/infra/wab/src/__tests/rateLimitPolicy.test.ts @@ -97,4 +97,12 @@ describe('rate-limit security policy', () => { limit: 1 })).toThrow(/must not exceed/) }) + + it.each(['-1', 'unlimited'])('allows an explicit %s rate-limit opt-out', value => { + process.env.TEST_RATE_LIMIT_MAX = value + expect(rateLimitOptions('TEST_RATE_LIMIT', { + windowMs: 60_000, + limit: 1 + }).limit).toBe(Number.MAX_SAFE_INTEGER) + }) }) diff --git a/infra/wab/src/app.ts b/infra/wab/src/app.ts index 6728e86d8..a9d97d2bf 100644 --- a/infra/wab/src/app.ts +++ b/infra/wab/src/app.ts @@ -1,89 +1,131 @@ -import express from "express" -import bodyParser from "body-parser" -import rateLimit from "express-rate-limit" -import { InfoController } from "./controllers/InfoController" -import { AuthController } from "./controllers/AuthController" -import { UserController } from "./controllers/UserController" -import { FaucetController } from "./controllers/FaucetController" -import { AccountDeletionController } from "./controllers/AccountDeletionController" -import { ShareController } from "./controllers/ShareController" -import { configureTrustProxy, rateLimitOptions } from "./security/rateLimitPolicy" +import express from 'express' +import bodyParser from 'body-parser' +import rateLimit from 'express-rate-limit' +import { InfoController } from './controllers/InfoController' +import { AuthController } from './controllers/AuthController' +import { UserController } from './controllers/UserController' +import { FaucetController } from './controllers/FaucetController' +import { AccountDeletionController } from './controllers/AccountDeletionController' +import { ShareController } from './controllers/ShareController' +import { configureTrustProxy, rateLimitOptions } from './security/rateLimitPolicy' import { bodyParserErrorHandler, concurrencyLimit, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceProfile, + responseSizeLimit, securityHeaders -} from "./security/edgePolicy" +} from './security/edgePolicy' const app = express() +const resourceProfile = readResourceProfile('WAB') app.disable('x-powered-by') configureTrustProxy(app) +app.use(initialDoubleSlashCompatibility) app.use(securityHeaders({ environmentPrefix: 'WAB' })) -app.use(corsPolicy({ - environmentPrefix: 'WAB', - methods: ['GET', 'POST', 'OPTIONS'] -})) -app.use(concurrencyLimit('WAB', 200)) -app.use(rateLimit(rateLimitOptions( - 'WAB_PRE_AUTH_RATE_LIMIT', - { windowMs: 60_000, limit: 300 } -))) -app.use(bodyParser.json({ - limit: readBodyLimitBytes('WAB', 256 * 1024) -})) +app.use( + corsPolicy({ + environmentPrefix: 'WAB', + methods: ['GET', 'POST', 'OPTIONS'] + }) +) +app.use( + concurrencyLimit( + 'WAB', + profileValue(resourceProfile, { + small: 64, + standard: 128, + highThroughput: 256 + }) + ) +) +app.use(rateLimit(rateLimitOptions('WAB_PRE_AUTH_RATE_LIMIT', { windowMs: 60_000, limit: 300 }))) +app.use( + bodyParser.json({ + limit: readBodyLimitBytes( + 'WAB', + profileValue(resourceProfile, { + small: 128 * 1024, + standard: 256 * 1024, + highThroughput: 1024 * 1024 + }) + ) + }) +) app.use(bodyParserErrorHandler) +app.use( + responseSizeLimit( + 'WAB', + profileValue(resourceProfile, { + small: 1024 * 1024, + standard: 2 * 1024 * 1024, + highThroughput: 8 * 1024 * 1024 + }) + ) +) -const authenticationLimiter = rateLimit(rateLimitOptions( - 'WAB_AUTH_RATE_LIMIT', - { windowMs: 15 * 60 * 1000, limit: 10 } -)) +const authenticationLimiter = rateLimit( + rateLimitOptions('WAB_AUTH_RATE_LIMIT', { windowMs: 15 * 60 * 1000, limit: 10 }) +) -const accountDeletionLimiter = rateLimit(rateLimitOptions( - 'WAB_ACCOUNT_DELETION_RATE_LIMIT', - { windowMs: 15 * 60 * 1000, limit: 5 } -)) +const accountDeletionLimiter = rateLimit( + rateLimitOptions('WAB_ACCOUNT_DELETION_RATE_LIMIT', { windowMs: 15 * 60 * 1000, limit: 5 }) +) -const userOperationLimiter = rateLimit(rateLimitOptions( - 'WAB_USER_RATE_LIMIT', - { windowMs: 15 * 60 * 1000, limit: 120 } -)) +const userOperationLimiter = rateLimit( + rateLimitOptions('WAB_USER_RATE_LIMIT', { windowMs: 15 * 60 * 1000, limit: 120 }) +) -const faucetLimiter = rateLimit(rateLimitOptions( - 'WAB_FAUCET_RATE_LIMIT', - { windowMs: 60 * 60 * 1000, limit: 5 } -)) +const faucetLimiter = rateLimit( + rateLimitOptions('WAB_FAUCET_RATE_LIMIT', { windowMs: 60 * 60 * 1000, limit: 5 }) +) -const shareLimiter = rateLimit(rateLimitOptions( - 'WAB_SHARE_RATE_LIMIT', - { windowMs: 15 * 60 * 1000, limit: 10 } -)) +const shareLimiter = rateLimit( + rateLimitOptions('WAB_SHARE_RATE_LIMIT', { windowMs: 15 * 60 * 1000, limit: 10 }) +) // Info route -app.get("/info", InfoController.getInfo) +app.get('/healthz', (_req, res) => { + res.setHeader('Cache-Control', 'no-store') + res.status(200).json({ + ok: true, + status: 'ok', + service: 'wab-server', + network: process.env.BSV_NETWORK ?? 'mainnet', + profile: resourceProfile + }) +}) +app.get('/info', InfoController.getInfo) // Auth routes -app.post("/auth/start", authenticationLimiter, AuthController.startAuth) -app.post("/auth/complete", authenticationLimiter, AuthController.completeAuth) +app.post('/auth/start', authenticationLimiter, AuthController.startAuth) +app.post('/auth/complete', authenticationLimiter, AuthController.completeAuth) // Account deletion routes (for users who can't access their account) // Rate limited to prevent SMS spam and brute-force attacks -app.post("/account/delete/start", accountDeletionLimiter, AccountDeletionController.startDeletion) -app.post("/account/delete/complete", accountDeletionLimiter, AccountDeletionController.completeDeletion) +app.post('/account/delete/start', accountDeletionLimiter, AccountDeletionController.startDeletion) +app.post( + '/account/delete/complete', + accountDeletionLimiter, + AccountDeletionController.completeDeletion +) // User routes -app.post("/user/linkedMethods", userOperationLimiter, UserController.listLinkedMethods) -app.post("/user/unlinkMethod", userOperationLimiter, UserController.unlinkMethod) -app.post("/user/delete", userOperationLimiter, UserController.deleteUser) +app.post('/user/linkedMethods', userOperationLimiter, UserController.listLinkedMethods) +app.post('/user/unlinkMethod', userOperationLimiter, UserController.unlinkMethod) +app.post('/user/delete', userOperationLimiter, UserController.deleteUser) // Faucet route -app.post("/faucet/request", faucetLimiter, FaucetController.requestFaucet) +app.post('/faucet/request', faucetLimiter, FaucetController.requestFaucet) // Shamir share routes (for 2-of-3 key recovery system) // Rate limited to prevent brute-force OTP attacks and share enumeration -app.post("/share/store", shareLimiter, ShareController.storeShare) -app.post("/share/retrieve", shareLimiter, ShareController.retrieveShare) -app.post("/share/update", shareLimiter, ShareController.updateShare) -app.post("/share/delete", shareLimiter, ShareController.deleteUser) +app.post('/share/store', shareLimiter, ShareController.storeShare) +app.post('/share/retrieve', shareLimiter, ShareController.retrieveShare) +app.post('/share/update', shareLimiter, ShareController.updateShare) +app.post('/share/delete', shareLimiter, ShareController.deleteUser) export default app diff --git a/infra/wab/src/knexfile.ts b/infra/wab/src/knexfile.ts index a84c05f1e..0e6ce15fd 100644 --- a/infra/wab/src/knexfile.ts +++ b/infra/wab/src/knexfile.ts @@ -1,6 +1,24 @@ import path from "node:path"; import { Knex } from "knex"; +function readPoolValue(name: string, fallback: number, allowZero = false): number { + const raw = process.env[name]; + if (raw == null || raw.trim() === "") return fallback; + const pattern = allowZero ? /^\d+$/ : /^[1-9]\d*$/; + if (!pattern.test(raw)) throw new Error(`${name} must be ${allowZero ? "a non-negative" : "a positive"} integer`); + const value = Number(raw); + if (!Number.isSafeInteger(value)) throw new Error(`${name} must be a safe integer`); + return value; +} + +const productionPool = { + min: readPoolValue("WAB_DB_POOL_MIN", 2, true), + max: readPoolValue("WAB_DB_POOL_MAX", 10) +}; +if (productionPool.min > productionPool.max) { + throw new Error("WAB_DB_POOL_MIN must not exceed WAB_DB_POOL_MAX"); +} + const connectionConfig = { user: process.env.DB_USER!, password: process.env.DB_PASS!, @@ -36,7 +54,7 @@ const config: { [key: string]: Knex.Config } = { production: { client: process.env.DB_CLIENT || "mysql2", connection: connectionConfig, - pool: { min: 2, max: 10 }, + pool: productionPool, migrations: { tableName: "knex_migrations" } diff --git a/infra/wab/src/security/edgePolicy.test.ts b/infra/wab/src/security/edgePolicy.test.ts index 1e169aee3..3d59d3882 100644 --- a/infra/wab/src/security/edgePolicy.test.ts +++ b/infra/wab/src/security/edgePolicy.test.ts @@ -5,13 +5,18 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readAllowedOrigins, readBodyLimitBytes, readCorsOriginSetting, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders } from './edgePolicy' -async function listen (app: express.Express): Promise<{ +async function listen(app: express.Express): Promise<{ server: Server origin: string }> { @@ -25,7 +30,7 @@ async function listen (app: express.Express): Promise<{ return { server, origin: `http://127.0.0.1:${address.port}` } } -async function close (server: Server): Promise { +async function close(server: Server): Promise { await new Promise((resolve, reject) => { server.close(error => { if (error != null) reject(error) @@ -47,10 +52,12 @@ describe('shared service edge policy', () => { delete process.env.CORS_MODE delete process.env.CORS_ALLOWED_ORIGINS const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -78,8 +85,9 @@ describe('shared service edge policy', () => { }) expect(preflight.status).toBe(204) expect(preflight.headers.get('access-control-allow-origin')).toBe('*') - expect(preflight.headers.get('access-control-allow-headers')) - .toContain('X-BSV-Action-Batch-Encoding') + expect(preflight.headers.get('access-control-allow-headers')).toContain( + 'X-BSV-Action-Batch-Encoding' + ) } finally { await close(server) } @@ -88,10 +96,12 @@ describe('shared service edge policy', () => { it('allows only explicitly configured browser origins', async () => { process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -124,10 +134,12 @@ describe('shared service edge policy', () => { delete process.env.TEST_CORS_ALLOWED_ORIGINS delete process.env.CORS_ALLOWED_ORIGINS const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -146,10 +158,12 @@ describe('shared service edge policy', () => { it('answers allowed preflight without wildcard policy', async () => { process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) const { server, origin } = await listen(app) try { @@ -170,16 +184,20 @@ describe('shared service edge policy', () => { it('rejects wildcard and malformed origin configuration', () => { process.env.TEST_CORS_ALLOWED_ORIGINS = '*' - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })).toThrow(/wildcard/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ).toThrow(/wildcard/) process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example/path' - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })).toThrow(/without paths/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ).toThrow(/without paths/) }) it('validates the complete CORS mode configuration matrix', () => { @@ -202,14 +220,8 @@ describe('shared service edge policy', () => { process.env.TEST_CORS_MODE = 'allowlist' process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example, https://wallet.example, https://wui.example' - expect(readAllowedOrigins('TEST')).toEqual([ - 'https://wallet.example', - 'https://wui.example' - ]) - expect(readCorsOriginSetting('TEST')).toEqual([ - 'https://wallet.example', - 'https://wui.example' - ]) + expect(readAllowedOrigins('TEST')).toEqual(['https://wallet.example', 'https://wui.example']) + expect(readCorsOriginSetting('TEST')).toEqual(['https://wallet.example', 'https://wui.example']) delete process.env.TEST_CORS_MODE delete process.env.TEST_CORS_ALLOWED_ORIGINS @@ -217,21 +229,27 @@ describe('shared service edge policy', () => { }) it('validates explicit origin and credential options', () => { - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowedOrigins: ['null'] - })).toThrow(/opaque/) - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowedOrigins: ['not an origin'] - })).toThrow(/invalid origin/) - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowCredentials: true - })).toThrow(/cookie credentials/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowedOrigins: ['null'] + }) + ).toThrow(/opaque/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowedOrigins: ['not an origin'] + }) + ).toThrow(/invalid origin/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowCredentials: true + }) + ).toThrow(/cookie credentials/) const disabled = corsPolicy({ environmentPrefix: 'TEST', @@ -277,11 +295,7 @@ describe('shared service edge policy', () => { sendStatus: jest.fn() } const next = jest.fn() - middleware( - { get: () => 'https://wallet.example', method: 'GET' } as any, - response as any, - next - ) + middleware({ get: () => 'https://wallet.example', method: 'GET' } as any, response as any, next) expect(headers.get('Vary')).toBe('Accept-Encoding, Origin') expect(headers.get('Access-Control-Allow-Origin')).toBe('https://wallet.example') expect(headers.get('Access-Control-Allow-Credentials')).toBe('true') @@ -296,10 +310,12 @@ describe('shared service edge policy', () => { process.env.TEST_CORS_MODE = 'allowlist' process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -342,15 +358,17 @@ describe('shared service edge policy', () => { process.env.TEST_STRICT_TRANSPORT_SECURITY = 'false' const app = express() app.enable('trust proxy') - app.use(securityHeaders({ - environmentPrefix: 'TEST', - contentSecurityPolicy: "default-src 'none'", - crossOriginResourcePolicy: 'same-origin', - crossOriginOpenerPolicy: 'same-origin', - frameOptions: 'DENY', - permissionsPolicy: 'camera=()', - strictTransportSecurity: true - })) + app.use( + securityHeaders({ + environmentPrefix: 'TEST', + contentSecurityPolicy: "default-src 'none'", + crossOriginResourcePolicy: 'same-origin', + crossOriginOpenerPolicy: 'same-origin', + frameOptions: 'DENY', + permissionsPolicy: 'camera=()', + strictTransportSecurity: true + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -375,10 +393,12 @@ describe('shared service edge policy', () => { delete process.env.TEST_CORS_ALLOWED_ORIGINS const app = express() app.use(securityHeaders({ environmentPrefix: 'TEST' })) - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -389,8 +409,9 @@ describe('shared service edge policy', () => { expect(response.status).toBe(200) expect(response.headers.get('access-control-allow-origin')).toBe('*') expect(response.headers.get('access-control-allow-credentials')).toBeNull() - expect(response.headers.get('content-security-policy')) - .toBe("default-src 'self'; connect-src https:") + expect(response.headers.get('content-security-policy')).toBe( + "default-src 'self'; connect-src https:" + ) } finally { await close(server) } @@ -519,7 +540,9 @@ describe('shared service edge policy', () => { app.use(concurrencyLimit('TEST', 10)) let releaseFirst: (() => void) | undefined app.get('/', async (_req, res) => { - await new Promise(resolve => { releaseFirst = resolve }) + await new Promise(resolve => { + releaseFirst = resolve + }) res.json({ ok: true }) }) const { server, origin } = await listen(app) @@ -528,7 +551,8 @@ describe('shared service edge policy', () => { headersTimeoutMs: 10_000, keepAliveTimeoutMs: 5_000, socketTimeoutMs: 30_000, - maxRequestsPerSocket: 100 + maxRequestsPerSocket: 100, + maxConnections: 50 }) try { @@ -545,8 +569,156 @@ describe('shared service edge policy', () => { expect(server.headersTimeout).toBe(10_000) expect(server.keepAliveTimeout).toBe(5_000) expect(server.maxRequestsPerSocket).toBe(100) + expect(server.maxConnections).toBe(50) } finally { await close(server) } }) + + it('honors explicit unlimited body, response, concurrency, and connection limits', () => { + process.env.TEST_MAX_BODY_BYTES = '-1' + expect(readBodyLimitBytes('TEST', 256)).toBe(Number.MAX_SAFE_INTEGER) + + process.env.TEST_MAX_RESPONSE_BYTES = 'unlimited' + const responseNext = jest.fn() + responseSizeLimit('TEST', 256)({} as any, {} as any, responseNext) + expect(responseNext).toHaveBeenCalledTimes(1) + + process.env.TEST_MAX_CONCURRENT_REQUESTS = '-1' + const concurrencyNext = jest.fn() + concurrencyLimit('TEST', 8)({} as any, {} as any, concurrencyNext) + expect(concurrencyNext).toHaveBeenCalledTimes(1) + + process.env.TEST_MAX_CONNECTIONS = '-1' + const server = { + setTimeout: jest.fn() + } as unknown as Server + configureHttpServer(server, 'TEST', { + requestTimeoutMs: 30_000, + headersTimeoutMs: 10_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 30_000, + maxRequestsPerSocket: 100, + maxConnections: 50 + }) + expect(server.maxConnections).toBe(Number.MAX_SAFE_INTEGER) + }) + + it('rejects invalid positive HTTP server settings', () => { + process.env.TEST_REQUEST_TIMEOUT_MS = '0' + const server = { + setTimeout: jest.fn() + } as unknown as Server + + expect(() => + configureHttpServer(server, 'TEST', { + requestTimeoutMs: 30_000, + headersTimeoutMs: 10_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 30_000, + maxRequestsPerSocket: 100, + maxConnections: 50 + }) + ).toThrow(/positive integer/) + }) + + it('selects tested resource profiles and explicit operator limits', () => { + expect(readResourceProfile('TEST')).toBe('standard') + process.env.TEST_RESOURCE_PROFILE = 'high-throughput' + expect(readResourceProfile('TEST')).toBe('high-throughput') + expect(profileValue('small', { small: 1, standard: 2, highThroughput: 3 })).toBe(1) + expect(profileValue('standard', { small: 1, standard: 2, highThroughput: 3 })).toBe(2) + expect(profileValue('high-throughput', { small: 1, standard: 2, highThroughput: 3 })).toBe(3) + + process.env.TEST_MAX_ITEMS = '1000' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(1_000) + process.env.TEST_MAX_ITEMS = '-1' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(-1) + process.env.TEST_MAX_ITEMS = 'unlimited' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(-1) + process.env.TEST_MAX_ITEMS = '0' + expect(() => readResourceLimit('TEST', 'MAX_ITEMS', 100)).toThrow(/positive integer/) + + process.env.TEST_RESOURCE_PROFILE = 'oversized' + expect(() => readResourceProfile('TEST')).toThrow(/small, standard, or high-throughput/) + }) + + it('tolerates only repeated initial slashes for compatibility', () => { + const next = jest.fn() + const request = { url: '///auth/start?mode=test' } + initialDoubleSlashCompatibility(request as any, {} as any, next) + expect(request.url).toBe('/auth/start?mode=test') + expect(next).toHaveBeenCalledTimes(1) + + const interior = { url: '/auth//start' } + initialDoubleSlashCompatibility(interior as any, {} as any, jest.fn()) + expect(interior.url).toBe('/auth//start') + }) + + it('rejects materialized responses above the configured byte budget', async () => { + process.env.TEST_MAX_RESPONSE_BYTES = '128' + const app = express() + app.use(responseSizeLimit('TEST', 1024)) + app.get('/small', (_req, res) => res.json({ ok: true })) + app.get('/large', (_req, res) => res.json({ value: 'x'.repeat(512) })) + const { server, origin } = await listen(app) + + try { + const small = await fetch(`${origin}/small`) + expect(small.status).toBe(200) + await expect(small.json()).resolves.toEqual({ ok: true }) + + const large = await fetch(`${origin}/large`) + expect(large.status).toBe(413) + await expect(large.json()).resolves.toMatchObject({ code: 'ERR_RESPONSE_TOO_LARGE' }) + } finally { + await close(server) + } + }) + + it.each([ + ['send', '12345', undefined], + ['send', Buffer.from('12345'), undefined], + ['send', new Uint8Array([1, 2, 3, 4, 5]), undefined], + ['send', { value: '12345' }, undefined], + ['end', '12345', 'utf8'], + ['end', Buffer.from('12345'), undefined], + ['end', new Uint8Array([1, 2, 3, 4, 5]), undefined], + ['end', { value: '12345' }, undefined] + ])('bounds every materialized %s response shape', (method, value, encoding) => { + process.env.TEST_MAX_RESPONSE_BYTES = '4' + let response: any + const originalEnd = jest.fn(() => response) + const originalSend = jest.fn((chunk: unknown) => { + response.end(chunk) + return response + }) + const originalJson = jest.fn((body: unknown) => { + response.send(JSON.stringify(body)) + return response + }) + response = { + status: jest.fn(() => response), + json: originalJson, + send: originalSend, + end: originalEnd + } + const next = jest.fn() + responseSizeLimit('TEST', 256)({} as any, response, next) + + if (method === 'send') response.send(value) + else response.end(value, encoding) + + expect(response.status).toHaveBeenCalledWith(413) + expect(originalJson).toHaveBeenCalledWith( + expect.objectContaining({ code: 'ERR_RESPONSE_TOO_LARGE' }) + ) + expect(originalSend).toHaveBeenCalled() + expect(originalEnd).toHaveBeenCalled() + + response.json({ ignored: true }) + response.send('ignored') + response.end('ignored') + expect(response.status).toHaveBeenCalledTimes(1) + }) }) diff --git a/infra/wab/src/security/edgePolicy.ts b/infra/wab/src/security/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/infra/wab/src/security/edgePolicy.ts +++ b/infra/wab/src/security/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/infra/wab/src/security/rateLimitPolicy.ts b/infra/wab/src/security/rateLimitPolicy.ts index dea60bac5..87aa8b78e 100644 --- a/infra/wab/src/security/rateLimitPolicy.ts +++ b/infra/wab/src/security/rateLimitPolicy.ts @@ -29,6 +29,16 @@ export function readBoundedInteger ( return parsed } +function readRateLimit (name: string, fallback: number): number { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const normalized = value.trim().toLowerCase() + // express-rate-limit has no disabled sentinel. A safe-integer ceiling is + // effectively unlimited while preserving a numeric value for its internals. + if (normalized === '-1' || normalized === 'unlimited') return Number.MAX_SAFE_INTEGER + return readBoundedInteger(name, fallback, MAX_RATE_LIMIT) +} + export function rateLimitOptions ( prefix: string, defaults: RateLimitDefaults, @@ -36,7 +46,7 @@ export function rateLimitOptions ( ): Partial { return { windowMs: readBoundedInteger(`${prefix}_WINDOW_MS`, defaults.windowMs, MAX_RATE_LIMIT_WINDOW_MS), - limit: readBoundedInteger(`${prefix}_MAX`, defaults.limit, MAX_RATE_LIMIT), + limit: readRateLimit(`${prefix}_MAX`, defaults.limit), standardHeaders: 'draft-8', legacyHeaders: false, handler: (_req: Request, res: Response) => { diff --git a/infra/wallet-infra/.env.example b/infra/wallet-infra/.env.example index 26af4b8df..0cb05de6e 100644 --- a/infra/wallet-infra/.env.example +++ b/infra/wallet-infra/.env.example @@ -8,6 +8,19 @@ TAAL_API_KEY= COMMISSION_FEE=0 COMMISSION_PUBLIC_KEY= FEE_MODEL={"model":"sat/kb","value":1} +WALLET_INFRA_ROLE=all + +# Provider options. WALLET_STORAGE_* names take precedence; historical names +# remain accepted. Admin keys and JSON settings may be raw or base64 encoded. +WALLET_STORAGE_TAAL_API_KEY= +WALLET_STORAGE_WHATSONCHAIN_API_KEY= +WALLET_STORAGE_BITAILS_API_KEY= +WALLET_STORAGE_ARCADE_URL= +WALLET_STORAGE_ARCADE_API_KEY= +WALLET_STORAGE_ARCADE_CALLBACK_TOKEN= +WALLET_STORAGE_EXCHANGE_RATES_API_KEY= +WALLET_STORAGE_GORILLAPOOL_ARC_ENABLED=true +# WALLET_STORAGE_ADMIN_IDENTITY_KEYS=02...,03... # OpenTelemetry. OTLP headers may contain credentials and must come from the # deployment secret manager. @@ -30,14 +43,28 @@ LOG_LEVEL=info # WALLET_STORAGE_STRICT_TRANSPORT_SECURITY=false # Bounded RPC/blob and connection policy. -WALLET_STORAGE_JSON_MAX_BODY_BYTES=31457280 +WALLET_STORAGE_RESOURCE_PROFILE=standard +WALLET_STORAGE_JSON_MAX_BODY_BYTES=8388608 WALLET_STORAGE_BINARY_MAX_BODY_BYTES=8388608 -WALLET_STORAGE_MAX_CONCURRENT_REQUESTS=200 +WALLET_STORAGE_MAX_RESPONSE_BYTES=8388608 +WALLET_STORAGE_MAX_CONCURRENT_REQUESTS=24 +WALLET_STORAGE_MAX_CONNECTIONS=1000 +WALLET_STORAGE_RPC_DEFAULT_LIST_LIMIT=1000 +WALLET_STORAGE_RPC_MAX_LIST_LIMIT=1000 +WALLET_STORAGE_RPC_MAX_ARRAY_ITEMS=1000000 +WALLET_STORAGE_RPC_MAX_RESPONSE_BYTES=8388608 WALLET_STORAGE_REQUEST_TIMEOUT_MS=120000 WALLET_STORAGE_HEADERS_TIMEOUT_MS=15000 WALLET_STORAGE_KEEP_ALIVE_TIMEOUT_MS=5000 WALLET_STORAGE_SOCKET_TIMEOUT_MS=120000 WALLET_STORAGE_MAX_REQUESTS_PER_SOCKET=1000 +WALLET_STORAGE_DB_POOL_MIN=2 +WALLET_STORAGE_DB_POOL_MAX=10 +WALLET_STORAGE_DB_CREATE_TIMEOUT_MS=10000 +WALLET_STORAGE_DB_ACQUIRE_TIMEOUT_MS=30000 +WALLET_STORAGE_DB_IDLE_TIMEOUT_MS=600000 +WALLET_STORAGE_DB_REAP_INTERVAL_MS=60000 +WALLET_STORAGE_DB_CREATE_RETRY_MS=200 # Rate-limit stores and proxy trust are configured through StorageServer # options. Replicated deployments must use a shared store and an explicit diff --git a/infra/wallet-infra/README.md b/infra/wallet-infra/README.md index 363eae0b5..739b939f2 100644 --- a/infra/wallet-infra/README.md +++ b/infra/wallet-infra/README.md @@ -4,6 +4,10 @@ This repository serves as a reference implementation for building and deploying Built on the [wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox), this implementation empowers developers with extensive customization options for authentication, monetization, and database management to name a few. +See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) +for RPC ceilings, API/monitor role separation, official-image provider settings, +and Wallet Storage HPA prerequisites. + ## Key Features 1. #### Out-of-the-Box UTXO Management diff --git a/infra/wallet-infra/src/KnexPaymentReplayStore.ts b/infra/wallet-infra/src/KnexPaymentReplayStore.ts new file mode 100644 index 000000000..f93d8b470 --- /dev/null +++ b/infra/wallet-infra/src/KnexPaymentReplayStore.ts @@ -0,0 +1,46 @@ +import type { PaymentReplayStore } from '@bsv/payment-express-middleware' +import type { Knex } from 'knex' + +export const PAYMENT_REPLAY_TABLE = 'payment_replays' + +function isDuplicate(error: unknown): boolean { + if (error == null || typeof error !== 'object') return false + const value = error as { code?: unknown; errno?: unknown } + return ( + value.code === 'ER_DUP_ENTRY' || + value.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || + value.code === 'SQLITE_CONSTRAINT_UNIQUE' || + value.errno === 1062 + ) +} + +/** Durable, replica-safe BRC-105 transaction replay claims. */ +export class KnexPaymentReplayStore implements PaymentReplayStore { + constructor( + private readonly knex: Knex, + private readonly ttlDays: number = 365 + ) { + if (!Number.isSafeInteger(ttlDays) || (ttlDays !== -1 && ttlDays < 1)) { + throw new TypeError('KnexPaymentReplayStore ttlDays must be -1 or a positive integer.') + } + } + + async claim(transactionId: string): Promise { + const now = new Date() + try { + await this.knex(PAYMENT_REPLAY_TABLE).insert({ + transactionId, + createdAt: now, + expiresAt: this.ttlDays === -1 ? null : new Date(now.getTime() + this.ttlDays * 24 * 60 * 60 * 1_000) + }) + return true + } catch (error) { + if (isDuplicate(error)) return false + throw error + } + } + + async pruneExpired(now = new Date()): Promise { + return await this.knex(PAYMENT_REPLAY_TABLE).whereNotNull('expiresAt').where('expiresAt', '<=', now).delete() + } +} diff --git a/infra/wallet-infra/src/index.ts b/infra/wallet-infra/src/index.ts index e770f2f98..0e888ebfb 100644 --- a/infra/wallet-infra/src/index.ts +++ b/infra/wallet-infra/src/index.ts @@ -8,6 +8,9 @@ import { StorageServer, Wallet, Monitor, + KnexSessionManager, + WalletLogger, + type WalletLoggerLevel, type WalletArgs } from '@bsv/wallet-toolbox' import knexPkg from 'knex' @@ -19,6 +22,7 @@ import { createRequire } from 'node:module' import packageJson from '../package.json' with { type: 'json' } import { trace, SpanStatusCode } from '@opentelemetry/api' import { log } from './logger.js' +import { KnexPaymentReplayStore } from './KnexPaymentReplayStore.js' import * as dotenv from 'dotenv' dotenv.config() @@ -64,11 +68,231 @@ const { SERVER_PRIVATE_KEY, KNEX_DB_CONNECTION, TAAL_API_KEY, + WHATSONCHAIN_API_KEY, + BITAILS_API_KEY, + ARCADE_URL, + ARCADE_API_KEY, + ARCADE_CALLBACK_TOKEN, + EXCHANGERATESAPI_KEY, COMMISSION_FEE = 0, COMMISSION_PUBLIC_KEY, - FEE_MODEL = '{"model":"sat/kb","value":1}' + FEE_MODEL = '{"model":"sat/kb","value":1}', + LOGGER_LEVEL } = process.env +type WalletInfraRole = 'all' | 'api' | 'monitor' + +function readPositiveInteger(name: string, fallback: number): number { + const raw = process.env[name] + if (raw == null || raw.trim() === '') return fallback + if (!/^[1-9]\d*$/.test(raw)) + throw new Error(`${name} must be a positive integer`) + const value = Number(raw) + if (!Number.isSafeInteger(value)) + throw new Error(`${name} must be a safe integer`) + return value +} + +function readNonNegativeInteger(name: string, fallback: number): number { + const raw = process.env[name] + if (raw == null || raw.trim() === '') return fallback + if (!/^\d+$/.test(raw)) + throw new Error(`${name} must be a non-negative integer`) + const value = Number(raw) + if (!Number.isSafeInteger(value)) + throw new Error(`${name} must be a safe integer`) + return value +} + +function readBoolean(name: string, fallback: boolean): boolean { + const raw = process.env[name] + if (raw == null || raw.trim() === '') return fallback + if (raw === 'true') return true + if (raw === 'false') return false + throw new Error(`${name} must be true or false`) +} + +function readRole(): WalletInfraRole { + const role = process.env.WALLET_INFRA_ROLE?.trim().toLowerCase() ?? 'all' + if (role !== 'all' && role !== 'api' && role !== 'monitor') { + throw new Error('WALLET_INFRA_ROLE must be all, api, or monitor') + } + return role +} + +function decodeJsonSetting(name: string, raw: string): string { + const trimmed = raw.trim() + const candidates = [trimmed] + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { + candidates.push(Buffer.from(trimmed, 'base64').toString('utf8').trim()) + } + for (const candidate of candidates) { + try { + JSON.parse(candidate) + return candidate + } catch { + // Try the next representation. + } + } + throw new Error(`${name} must contain JSON or base64-encoded JSON`) +} + +function readAdminIdentityKeys(): string[] | undefined { + const raw = + process.env.WALLET_STORAGE_ADMIN_IDENTITY_KEYS ?? + process.env.ADMIN_IDENTITY_KEYS + if (raw == null || raw.trim() === '') return undefined + const direct = raw.split(',').map(value => value.trim()) + const decoded = direct.every(value => /^(02|03)[0-9a-fA-F]{64}$/.test(value)) + ? direct + : Buffer.from(raw, 'base64') + .toString('utf8') + .split(',') + .map(value => value.trim()) + if (!decoded.every(value => /^(02|03)[0-9a-fA-F]{64}$/.test(value))) { + throw new Error( + 'WALLET_STORAGE_ADMIN_IDENTITY_KEYS must contain comma-separated compressed public keys or their base64 encoding' + ) + } + return [...new Set(decoded.map(value => value.toLowerCase()))] +} + +function readWalletLoggerLevel(): WalletLoggerLevel | undefined { + const raw = LOGGER_LEVEL?.trim().toLowerCase() + if (raw == null || raw === '') return undefined + if (!['error', 'warn', 'info', 'debug', 'trace'].includes(raw)) { + throw new Error('LOGGER_LEVEL must be error, warn, info, debug, or trace') + } + return raw as WalletLoggerLevel +} + +type WalletChain = 'main' | 'test' | 'ttn' | 'tstn' | 'mock' + +function readWalletChain(): WalletChain { + const allowedChains: WalletChain[] = ['main', 'test', 'ttn', 'tstn', 'mock'] + if ( + typeof BSV_NETWORK === 'string' && + allowedChains.includes(BSV_NETWORK as WalletChain) + ) { + return BSV_NETWORK as WalletChain + } + log.warn( + { + operation: 'chain.select', + bsv_network: BSV_NETWORK, + fallback_chain: 'main' + }, + 'Invalid BSV_NETWORK value provided, falling back to main' + ) + return 'main' +} + +function configuredServiceOptions(chain: Exclude) { + const options = Services.createDefaultOptions(chain) + if (providerConfig.taalApiKey) { + options.arcConfig.apiKey = providerConfig.taalApiKey + options.taalApiKey = providerConfig.taalApiKey + } + if (providerConfig.whatsOnChainApiKey) { + options.whatsOnChainApiKey = providerConfig.whatsOnChainApiKey + } + if (providerConfig.bitailsApiKey) + options.bitailsApiKey = providerConfig.bitailsApiKey + if (providerConfig.exchangeRatesApiKey) { + options.exchangeratesapiKey = providerConfig.exchangeRatesApiKey + } + if (process.env.WALLET_STORAGE_TAAL_ARC_URL) { + options.arcUrl = process.env.WALLET_STORAGE_TAAL_ARC_URL + } + let gorillaPoolDefault = true + if (process.env.GORILLAPOOL_ARC_ENABLED != null) { + gorillaPoolDefault = readBoolean('GORILLAPOOL_ARC_ENABLED', true) + } + const gorillaPoolEnabled = readBoolean( + 'WALLET_STORAGE_GORILLAPOOL_ARC_ENABLED', + gorillaPoolDefault + ) + if (!gorillaPoolEnabled) options.arcGorillaPoolUrl = undefined + if (process.env.WALLET_STORAGE_GORILLAPOOL_ARC_URL) { + options.arcGorillaPoolUrl = process.env.WALLET_STORAGE_GORILLAPOOL_ARC_URL + } + if (providerConfig.arcadeUrl) { + options.arcadeUrl = providerConfig.arcadeUrl + options.arcadeConfig = { + apiKey: providerConfig.arcadeApiKey || undefined, + callbackToken: providerConfig.arcadeCallbackToken || undefined + } + } + return options +} + +async function createServicesAndMonitorOptions( + chain: WalletChain, + knex: Knex, + storage: WalletStorageManager +) { + if (chain === 'mock') { + const services = new MockServices(knex) + await services.initialize() + return { + services, + monitorOptions: { + chain, + services, + storage, + chaintracks: services.tracker, + msecsWaitPerMerkleProofServiceReq: 500, + taskRunWaitMsecs: 5000, + abandonedMsecs: 1000 * 60 * 5, + unprovenAttemptsLimitTest: 10, + unprovenAttemptsLimitMain: 144, + maxRebroadcastAttempts: 0 + } + } + } + const services = new Services(configuredServiceOptions(chain)) + return { + services, + monitorOptions: Monitor.createDefaultWalletMonitorOptions( + chain, + storage, + services + ) + } +} + +function walletNetworkPreset( + chain: WalletChain +): 'local' | 'mainnet' | 'testnet' { + if (chain === 'main') return 'mainnet' + if (chain === 'test') return 'testnet' + return 'local' +} + +function createWalletLoggerFactory() { + const loggerLevel = readWalletLoggerLevel() + if (loggerLevel == null) return undefined + return (source?: string | import('@bsv/sdk').WalletLoggerInterface) => { + const logger = new WalletLogger(source) + logger.level = loggerLevel + logger.flushFormat = 'json' as const + return logger + } +} + +const providerConfig = { + taalApiKey: process.env.WALLET_STORAGE_TAAL_API_KEY ?? TAAL_API_KEY, + whatsOnChainApiKey: + process.env.WALLET_STORAGE_WHATSONCHAIN_API_KEY ?? WHATSONCHAIN_API_KEY, + bitailsApiKey: process.env.WALLET_STORAGE_BITAILS_API_KEY ?? BITAILS_API_KEY, + arcadeUrl: process.env.WALLET_STORAGE_ARCADE_URL ?? ARCADE_URL, + arcadeApiKey: process.env.WALLET_STORAGE_ARCADE_API_KEY ?? ARCADE_API_KEY, + arcadeCallbackToken: + process.env.WALLET_STORAGE_ARCADE_CALLBACK_TOKEN ?? ARCADE_CALLBACK_TOKEN, + exchangeRatesApiKey: + process.env.WALLET_STORAGE_EXCHANGE_RATES_API_KEY ?? EXCHANGERATESAPI_KEY +} + async function setupWalletStorageAndMonitor(): Promise<{ databaseName: string knex: Knex @@ -100,7 +324,9 @@ async function setupWalletStorageAndMonitor(): Promise<{ ) } // Parse database connection details - const connection = JSON.parse(KNEX_DB_CONNECTION) + const connection = JSON.parse( + decodeJsonSetting('KNEX_DB_CONNECTION', KNEX_DB_CONNECTION) + ) const databaseName = connection['database'] // You can also use an imported knex configuration file. @@ -109,37 +335,39 @@ async function setupWalletStorageAndMonitor(): Promise<{ connection, useNullAsDefault: true, pool: { - min: 2, - max: 10, - createTimeoutMillis: 10000, - acquireTimeoutMillis: 30000, - idleTimeoutMillis: 600000, - reapIntervalMillis: 60000, - createRetryIntervalMillis: 200, + min: readNonNegativeInteger('WALLET_STORAGE_DB_POOL_MIN', 2), + max: readPositiveInteger('WALLET_STORAGE_DB_POOL_MAX', 10), + createTimeoutMillis: readPositiveInteger( + 'WALLET_STORAGE_DB_CREATE_TIMEOUT_MS', + 10_000 + ), + acquireTimeoutMillis: readPositiveInteger( + 'WALLET_STORAGE_DB_ACQUIRE_TIMEOUT_MS', + 30_000 + ), + idleTimeoutMillis: readPositiveInteger( + 'WALLET_STORAGE_DB_IDLE_TIMEOUT_MS', + 600_000 + ), + reapIntervalMillis: readPositiveInteger( + 'WALLET_STORAGE_DB_REAP_INTERVAL_MS', + 60_000 + ), + createRetryIntervalMillis: readPositiveInteger( + 'WALLET_STORAGE_DB_CREATE_RETRY_MS', + 200 + ), propagateCreateError: false } } - const knex = makeKnex(knexConfig) - - // Select chain from BSV_NETWORK: "main", "test", "ttn" (TeraTestNet), - // "tstn" (Teranode Scaling Test Net), or "mock" (defaults to "main") - const allowedChains = ['main', 'test', 'ttn', 'tstn', 'mock'] as const - let chain: (typeof allowedChains)[number] = 'main' - if ( - typeof BSV_NETWORK === 'string' && - allowedChains.includes(BSV_NETWORK as any) - ) { - chain = BSV_NETWORK as (typeof allowedChains)[number] - } else if (BSV_NETWORK !== 'main') { - log.warn( - { - operation: 'chain.select', - bsv_network: BSV_NETWORK, - fallback_chain: 'main' - }, - 'Invalid BSV_NETWORK value provided, falling back to main' + if ((knexConfig.pool?.min ?? 0) > (knexConfig.pool?.max ?? 0)) { + throw new Error( + 'WALLET_STORAGE_DB_POOL_MIN must not exceed WALLET_STORAGE_DB_POOL_MAX' ) } + const knex = makeKnex(knexConfig) + + const chain = readWalletChain() // Initialize storage components const rootKey = PrivateKey.fromHex(SERVER_PRIVATE_KEY) @@ -150,7 +378,7 @@ async function setupWalletStorageAndMonitor(): Promise<{ knex, commissionSatoshis, commissionPubKeyHex: COMMISSION_PUBLIC_KEY || undefined, - feeModel: JSON.parse(FEE_MODEL) + feeModel: JSON.parse(decodeJsonSetting('FEE_MODEL', String(FEE_MODEL))) }) await activeStorage.migrate(databaseName, storageIdentityKey) @@ -162,54 +390,16 @@ async function setupWalletStorageAndMonitor(): Promise<{ ) await storage.makeAvailable() - // Initialize wallet components - let services - let monopts - if (chain === 'mock') { - services = new MockServices(knex) - await services.initialize() - monopts = { - chain, - services, - storage, - chaintracks: services.tracker, - msecsWaitPerMerkleProofServiceReq: 500, - taskRunWaitMsecs: 5000, - abandonedMsecs: 1000 * 60 * 5, - unprovenAttemptsLimitTest: 10, - unprovenAttemptsLimitMain: 144, - maxRebroadcastAttempts: 0 - } - } else { - const servOpts = Services.createDefaultOptions(chain) - if (TAAL_API_KEY) { - servOpts.arcConfig.apiKey = TAAL_API_KEY - servOpts.taalApiKey = TAAL_API_KEY - } - services = new Services(servOpts) - monopts = Monitor.createDefaultWalletMonitorOptions( - chain, - storage, - services - ) - } + const { services, monitorOptions } = await createServicesAndMonitorOptions( + chain, + knex, + storage + ) const keyDeriver = new KeyDeriver(rootKey) - const monitor = new Monitor(monopts) + const monitor = new Monitor(monitorOptions) monitor.addDefaultTasks() - let networkPresetForLookupResolver: 'local' | 'mainnet' | 'testnet' = - 'local' - switch (chain) { - case 'main': - networkPresetForLookupResolver = 'mainnet' - break - case 'test': - networkPresetForLookupResolver = 'testnet' - break - default: - break - } const wallet = new Wallet({ chain, keyDeriver, @@ -217,18 +407,36 @@ async function setupWalletStorageAndMonitor(): Promise<{ services, monitor, lookupResolver: new LookupResolver({ - networkPreset: networkPresetForLookupResolver + networkPreset: walletNetworkPreset(chain) }) }) + const makeLogger = createWalletLoggerFactory() + // Set up server options - const serverOptions: WalletStorageServerOptions = { + const serverOptions: WalletStorageServerOptions & { + paymentReplayStore: KnexPaymentReplayStore + } = { port: Number(HTTP_PORT), wallet, - monetize: false, - calculateRequestPrice: async () => { - return 0 // Monetize your server here! Price is in satoshis. - } + monetize: readBoolean('WALLET_STORAGE_MONETIZATION_ENABLED', false), + calculateRequestPrice: () => + readNonNegativeInteger('WALLET_STORAGE_PRICE_SATOSHIS', 100), + adminIdentityKeys: readAdminIdentityKeys(), + makeLogger, + sessionManager: new KnexSessionManager(knex, { + ttlMs: readPositiveInteger( + 'WALLET_STORAGE_AUTH_SESSION_TTL_MS', + 24 * 60 * 60 * 1_000 + ) + }), + paymentReplayStore: new KnexPaymentReplayStore( + knex, + process.env.WALLET_STORAGE_PAYMENT_REPLAY_TTL_DAYS === '-1' + ? -1 + : readPositiveInteger('WALLET_STORAGE_PAYMENT_REPLAY_TTL_DAYS', 365) + ), + logRpcRequests: readBoolean('WALLET_STORAGE_LOG_RPC_REQUESTS', true) } const server = new StorageServer(activeStorage, serverOptions) @@ -258,6 +466,7 @@ async function setupWalletStorageAndMonitor(): Promise<{ await tracer.startActiveSpan('wallet-infra.bootstrap', async span => { const startedAt = Date.now() try { + const role = readRole() const walletToolboxVersion = String( packageJson.dependencies['@bsv/wallet-toolbox'] ).replace(/^[~^]/, '') @@ -275,18 +484,22 @@ await tracer.startActiveSpan('wallet-infra.bootstrap', async span => { 'storage settings' ) - context.server.start() - log.info( - { operation: 'storage_server.start', outcome: 'ok' }, - 'StorageServer started' - ) + if (role !== 'monitor') { + context.server.start() + log.info( + { operation: 'storage_server.start', outcome: 'ok' }, + 'StorageServer started' + ) + } - await context.monitor.startTasks() - log.info({ operation: 'monitor.start', outcome: 'ok' }, 'Monitor started') + if (role !== 'api') { + await context.monitor.startTasks() + log.info({ operation: 'monitor.start', outcome: 'ok' }, 'Monitor started') + } // Conditionally start nginx let nginxProcess: ChildProcess | undefined - if (ENABLE_NGINX === 'true') { + if (role !== 'monitor' && ENABLE_NGINX === 'true') { nginxProcess = spawn('/usr/sbin/nginx', [], { stdio: ['inherit', 'inherit', 'inherit'] }) @@ -299,9 +512,11 @@ await tracer.startActiveSpan('wallet-infra.bootstrap', async span => { { operation: 'shutdown', signal }, 'wallet-infra shutdown started' ) - context.monitor.stopTasks() + if (role !== 'api') context.monitor.stopTasks() nginxProcess?.kill('SIGTERM') - await closeHttpServer(context.server.server as Server) + if (role !== 'monitor' && context.server.server != null) { + await closeHttpServer(context.server.server as Server) + } await context.wallet.destroy() log.info( { operation: 'shutdown', outcome: 'ok', signal }, @@ -322,6 +537,7 @@ await tracer.startActiveSpan('wallet-infra.bootstrap', async span => { const duration_ms = Date.now() - startedAt span.setAttribute('bsv.network', String(BSV_NETWORK)) span.setAttribute('nginx.enabled', ENABLE_NGINX === 'true') + span.setAttribute('wallet.infra.role', role) span.setStatus({ code: SpanStatusCode.OK }) log.info( { operation: 'bootstrap', outcome: 'ok', duration_ms }, diff --git a/package.json b/package.json index 6004c3d05..610a10161 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "sync-versions": "node scripts/sync-versions.mjs", "sync:service-rate-limit-policy": "node scripts/sync-service-rate-limit-policy.mjs", "sync:service-edge-policy": "node scripts/sync-service-edge-policy.mjs", + "sync:service-runtime-copies": "node scripts/sync-service-runtime-copies.mjs", + "resource-profiles:check": "node scripts/benchmark-service-resource-profiles.mjs --check", "license:sync": "node scripts/package-license-policy.mjs --write", "license:check": "node scripts/package-license-policy.mjs", "license:pack-check": "node scripts/check-package-license-tarballs.mjs", @@ -20,7 +22,7 @@ "check-versions": "node scripts/check-versions.mjs", "typescript:check": "node scripts/typescript-toolchain.mjs", "health:baseline": "node scripts/repository-health.mjs --update-contract-baseline", - "health:check": "node --test scripts/*.test.mjs && pnpm contributor-policy:check && pnpm docs:facts:check && pnpm ops:check && node scripts/dependency-release-governance.mjs check && node scripts/browser-artifact-governance.mjs && node scripts/typescript-toolchain.mjs && node scripts/package-license-policy.mjs && node scripts/sync-service-rate-limit-policy.mjs --check && node scripts/sync-service-edge-policy.mjs --check && node scripts/repository-health.mjs", + "health:check": "node --test scripts/*.test.mjs && pnpm contributor-policy:check && pnpm docs:facts:check && pnpm ops:check && pnpm resource-profiles:check && node scripts/dependency-release-governance.mjs check && node scripts/browser-artifact-governance.mjs && node scripts/typescript-toolchain.mjs && node scripts/package-license-policy.mjs && node scripts/sync-service-rate-limit-policy.mjs --check && node scripts/sync-service-edge-policy.mjs --check && node scripts/sync-service-runtime-copies.mjs --check && node scripts/repository-health.mjs", "health:report": "node scripts/repository-health.mjs --format markdown", "contributor-policy:check": "node scripts/contributor-policy.mjs", "contributor-policy:sync": "node scripts/contributor-policy.mjs --write", diff --git a/packages/messaging/message-box-client/README.md b/packages/messaging/message-box-client/README.md index 8a34ce48a..d00c43a4a 100644 --- a/packages/messaging/message-box-client/README.md +++ b/packages/messaging/message-box-client/README.md @@ -43,6 +43,27 @@ await messages.acknowledgeMessage({ }) ``` +`listMessages()` preserves its historical fetch-all behavior by following +bounded server pages. Limit aggregate client memory when appropriate: + +```ts +const firstTwoPages = await messages.listMessages({ + messageBox: 'general_inbox', + pageSize: 250, + maxPages: 2 +}) + +const nextThousand = await messages.listMessages({ + messageBox: 'general_inbox', + offset: 1000, + limit: 1000 +}) +``` + +The client uses AuthFetch for BRC-105 challenges. It intentionally does not add +a second Message Box-specific cost-approval mechanism because BRC-100 wallet +permissions already govern payment authorization. + Explicit `init()` is optional. Public methods initialize the wallet identity when needed: diff --git a/packages/messaging/message-box-client/package.json b/packages/messaging/message-box-client/package.json index 8208c8ba3..cf71caba0 100644 --- a/packages/messaging/message-box-client/package.json +++ b/packages/messaging/message-box-client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/message-box-client", - "version": "2.2.6", + "version": "2.3.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/messaging/message-box-client/src/MessageBoxClient.ts b/packages/messaging/message-box-client/src/MessageBoxClient.ts index 07bb7669e..58c6d0f01 100644 --- a/packages/messaging/message-box-client/src/MessageBoxClient.ts +++ b/packages/messaging/message-box-client/src/MessageBoxClient.ts @@ -1746,24 +1746,31 @@ export class MessageBoxClient { async listMessages({ messageBox, host, - acceptPayments + acceptPayments, + offset, + skip, + limit, + pageSize, + maxPages }: ListMessagesParams): Promise { const shouldAcceptPayments = acceptPayments !== false if (typeof messageBox !== 'string' || messageBox.trim() === '') { throw new Error('MessageBox cannot be empty') } - let hosts: string[] = host != null ? [normalizeMessageBoxHost(host)] : [] - if (hosts.length === 0) { - const advertisedHosts = await this.queryAdvertisements(await this.getIdentityKey()) - hosts = Array.from(new Set([this.host, ...advertisedHosts.map(h => h.host)])) - } + const hosts = await this.resolveMessageHosts(host) // Query each host in parallel const fetchFromHost = async (host: string): Promise => { try { Logger.log(`[MB CLIENT] Listing messages from ${host}…`) - return await this.fetchMessagePages(host, messageBox) + return await this.fetchMessagePages(host, messageBox, { + offset, + skip, + limit, + pageSize, + maxPages + }) } catch (err) { Logger.log(`[MB CLIENT DEBUG] listMessages failed for ${host}:`, err) throw err // re-throw to be caught in the settled promise @@ -1797,7 +1804,8 @@ export class MessageBoxClient { // 6. Early‑out: no messages but at least one host succeeded → [] if (dedupMap.size === 0) return [] - const messages: PeerMessage[] = Array.from(dedupMap.values()) + const deduplicated = Array.from(dedupMap.values()) + const messages: PeerMessage[] = limit == null ? deduplicated : deduplicated.slice(0, limit) const parsed = messages.map(message => this.parseMessageEnvelope(message)) @@ -1822,6 +1830,14 @@ export class MessageBoxClient { return messages } + private async resolveMessageHosts(host?: string): Promise { + if (host != null) return [normalizeMessageBoxHost(host)] + const advertisedHosts = await this.queryAdvertisements(await this.getIdentityKey()) + return Array.from( + new Set([this.host, ...advertisedHosts.map(advertisement => advertisement.host)]) + ) + } + /** * @method listMessagesLite * @async @@ -1853,12 +1869,26 @@ export class MessageBoxClient { * }) * console.log(messages) */ - async listMessagesLite({ messageBox, host }: ListMessagesParams): Promise { + async listMessagesLite({ + messageBox, + host, + offset, + skip, + limit, + pageSize, + maxPages + }: ListMessagesParams): Promise { if (typeof messageBox !== 'string' || messageBox.trim() === '') { throw new Error('MessageBox cannot be empty') } const finalHost = normalizeMessageBoxHost(host ?? this.host) - const messages = await this.fetchMessagePages(finalHost, messageBox) + const messages = await this.fetchMessagePages(finalHost, messageBox, { + offset, + skip, + limit, + pageSize, + maxPages + }) await this.mapWithConcurrency(messages, 4, async message => { try { @@ -1895,41 +1925,130 @@ export class MessageBoxClient { return messages } - private async fetchMessagePages(host: string, messageBox: string): Promise { - const pageSize = 1_000 - const maximumPages = 100 + private async fetchMessagePages( + host: string, + messageBox: string, + options: Pick = {} + ): Promise { + const { startingOffset, totalLimit, requestedPageSize, maximumPages } = + this.normalizeMessagePageOptions(options) const messages: PeerMessage[] = [] - - for (let page = 0; page < maximumPages; page++) { - const offset = page * pageSize - const res = await this.authFetch.fetch(messageBoxEndpoint(host, '/listMessages'), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ messageBox, limit: pageSize, offset }) - }) - if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`) - - const data = await res.json() - if (data.status === 'error') { - throw new Error(data.description ?? 'Unknown server error') - } - if (!Array.isArray(data.messages)) { - throw new TypeError('Message Box server returned an invalid messages payload') - } - messages.push(...(data.messages as PeerMessage[])) + let offset = startingOffset + let page = 0 + + while (maximumPages === -1 || page < maximumPages) { + const remaining = totalLimit == null ? undefined : totalLimit - messages.length + if (remaining != null && remaining <= 0) return messages + const pageLimit = this.messagePageLimit(requestedPageSize, remaining) + const { data, pageMessages } = await this.fetchMessagePage( + host, + messageBox, + offset, + pageLimit + ) + const accepted = remaining == null ? pageMessages : pageMessages.slice(0, remaining) + messages.push(...accepted) // Legacy Message Box servers returned the complete collection without // pagination metadata and may ignore limit/offset. Only continue when a // pagination-aware server explicitly advertises another page. if (data.hasMore !== true) return messages + offset = this.nextMessagePageOffset(data, pageMessages.length, offset, requestedPageSize) + page += 1 } throw new Error( - `Message Box pagination exceeded ${maximumPages * pageSize} messages; ` + - 'acknowledge messages or request smaller application-level batches.' + `Message Box pagination exceeded ${maximumPages} pages; ` + + 'acknowledge messages, raise maxPages, or set an application-level limit.' ) } + private normalizeMessagePageOptions( + options: Pick + ): { + startingOffset: number + totalLimit?: number + requestedPageSize?: number + maximumPages: number + } { + if (options.offset != null && options.skip != null && options.offset !== options.skip) { + throw new RangeError('offset and skip must match when both are provided') + } + const startingOffset = options.offset ?? options.skip ?? 0 + const maximumPages = options.maxPages ?? -1 + this.assertMessagePageOption('offset', startingOffset, 0) + this.assertMessagePageOption('limit', options.limit, 1) + this.assertMessagePageOption('pageSize', options.pageSize, 1) + this.assertMessagePageOption('maxPages', maximumPages, 1, true) + return { + startingOffset, + totalLimit: options.limit, + requestedPageSize: options.pageSize, + maximumPages + } + } + + private assertMessagePageOption( + name: string, + value: number | undefined, + minimum: number, + allowUnlimited: boolean = false + ): void { + if (value == null) return + const unlimited = allowUnlimited && value === -1 + if (Number.isSafeInteger(value) && (unlimited || value >= minimum)) return + const unlimitedPrefix = allowUnlimited ? '-1 or ' : '' + const sign = minimum === 0 ? 'non-negative' : 'positive' + throw new RangeError(`${name} must be ${unlimitedPrefix}a ${sign} safe integer`) + } + + private messagePageLimit( + requestedPageSize: number | undefined, + remaining: number | undefined + ): number | undefined { + if (requestedPageSize == null) return remaining + if (remaining == null) return requestedPageSize + return Math.min(requestedPageSize, remaining) + } + + private async fetchMessagePage( + host: string, + messageBox: string, + offset: number, + limit: number | undefined + ): Promise<{ data: any; pageMessages: PeerMessage[] }> { + const body: Record = { messageBox, offset } + if (limit != null) body.limit = limit + const response = await this.authFetch.fetch(messageBoxEndpoint(host, '/listMessages'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`) + + const data = await response.json() + if (data.status === 'error') throw new Error(data.description ?? 'Unknown server error') + if (!Array.isArray(data.messages)) { + throw new TypeError('Message Box server returned an invalid messages payload') + } + return { data, pageMessages: data.messages as PeerMessage[] } + } + + private nextMessagePageOffset( + data: any, + pageMessageCount: number, + currentOffset: number, + requestedPageSize: number | undefined + ): number { + const nextOffset = Number(data.nextOffset) + if (Number.isSafeInteger(nextOffset) && nextOffset > currentOffset) return nextOffset + if (pageMessageCount > 0) return currentOffset + pageMessageCount + + const serverLimit = Number(data.limit) + if (Number.isSafeInteger(serverLimit) && serverLimit > 0) return currentOffset + serverLimit + return currentOffset + (requestedPageSize ?? 1_000) + } + /** * @method tryParse * @private diff --git a/packages/messaging/message-box-client/src/__tests/MessageBoxClientHardening.test.ts b/packages/messaging/message-box-client/src/__tests/MessageBoxClientHardening.test.ts index f16fe3893..1358b1da7 100644 --- a/packages/messaging/message-box-client/src/__tests/MessageBoxClientHardening.test.ts +++ b/packages/messaging/message-box-client/src/__tests/MessageBoxClientHardening.test.ts @@ -494,18 +494,75 @@ describe('MessageBoxClient hardening branches', () => { ).rejects.toThrow(error) }) - it('bounds pagination even if a server continually claims another page', async () => { + it('allows callers to bound pagination when a server continually claims another page', async () => { fetchMock.mockResolvedValue(jsonResponse({ status: 'success', messages: [], hasMore: true })) await expect( client.listMessagesLite({ messageBox: 'inbox', - host: 'https://message-box.example/api' + host: 'https://message-box.example/api', + maxPages: 100 }) - ).rejects.toThrow('pagination exceeded 100000 messages') + ).rejects.toThrow('pagination exceeded 100 pages') expect(fetchMock).toHaveBeenCalledTimes(100) }) + it('uses server-default pages while honoring client skip and total limits', async () => { + fetchMock + .mockResolvedValueOnce( + jsonResponse({ + status: 'success', + messages: [{ messageId: 'one', sender: recipientA, body: 'one' }], + hasMore: true, + nextOffset: 6 + }) + ) + .mockResolvedValueOnce( + jsonResponse({ + status: 'success', + messages: [ + { messageId: 'two', sender: recipientA, body: 'two' }, + { messageId: 'ignored', sender: recipientA, body: 'ignored' } + ], + hasMore: true, + nextOffset: 8 + }) + ) + + await expect( + client.listMessagesLite({ + messageBox: 'inbox', + host: 'https://message-box.example/api', + skip: 5, + limit: 2 + }) + ).resolves.toEqual([ + expect.objectContaining({ messageId: 'one' }), + expect.objectContaining({ messageId: 'two' }) + ]) + + expect(JSON.parse(fetchMock.mock.calls[0][1]?.body as string)).toEqual({ + messageBox: 'inbox', + offset: 5, + limit: 2 + }) + expect(JSON.parse(fetchMock.mock.calls[1][1]?.body as string)).toEqual({ + messageBox: 'inbox', + offset: 6, + limit: 1 + }) + }) + + it('rejects ambiguous or unsafe pagination controls before network work', async () => { + await expect( + client.listMessagesLite({ messageBox: 'inbox', offset: 1, skip: 2 }) + ).rejects.toThrow('offset and skip must match') + await expect(client.listMessagesLite({ messageBox: 'inbox', limit: 0 })).rejects.toThrow( + 'limit must be' + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + it('marks an individual message when decryption fails', async () => { wallet.decrypt.mockRejectedValue(new Error('cannot decrypt')) fetchMock.mockResolvedValue( diff --git a/packages/messaging/message-box-client/src/__tests/MessageBoxClientPermissions.test.ts b/packages/messaging/message-box-client/src/__tests/MessageBoxClientPermissions.test.ts index 21d6b8cd9..308adce95 100644 --- a/packages/messaging/message-box-client/src/__tests/MessageBoxClientPermissions.test.ts +++ b/packages/messaging/message-box-client/src/__tests/MessageBoxClientPermissions.test.ts @@ -202,7 +202,6 @@ describe('MessageBoxClient permission contract', () => { expect(fetchMock).toHaveBeenCalledTimes(2) expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toEqual({ messageBox: 'inbox', - limit: 1_000, offset: 1_000 }) }) diff --git a/packages/messaging/message-box-client/src/types.ts b/packages/messaging/message-box-client/src/types.ts index 607629a6f..b8b76c2ff 100644 --- a/packages/messaging/message-box-client/src/types.ts +++ b/packages/messaging/message-box-client/src/types.ts @@ -118,6 +118,16 @@ export interface ListMessagesParams { messageBox: string host?: string acceptPayments?: boolean + /** Starting message offset. `skip` is an equivalent compatibility alias. */ + offset?: number + /** Compatibility alias for `offset`; both values must match when supplied together. */ + skip?: number + /** Maximum total messages accumulated by this call. Omit to retain fetch-all behavior. */ + limit?: number + /** Per-request server page size. Omit to use the server operator's configured default. */ + pageSize?: number + /** Optional pagination safety ceiling. Defaults to -1 to preserve fetch-all compatibility. */ + maxPages?: number } /** diff --git a/packages/middleware/auth-express-middleware/API.md b/packages/middleware/auth-express-middleware/API.md index 97302d734..e87b046d4 100644 --- a/packages/middleware/auth-express-middleware/API.md +++ b/packages/middleware/auth-express-middleware/API.md @@ -32,6 +32,7 @@ export interface AuthMiddlewareOptions { logger?: typeof console logLevel?: LogLevel transportLimits?: Partial + telemetry?: TelemetryConfig } ``` @@ -65,6 +66,15 @@ Optional logger (e.g., console). If not provided, logging is disabled. logger?: typeof console ``` +#### Property telemetry + +Optional provider-neutral authentication timing. Header values, signatures, +certificate contents, wallet data, and peer identities are never emitted. + +```ts +telemetry?: TelemetryConfig +``` + #### Property transportLimits Bounds unauthenticated work and pending protocol state. Defaults to a @@ -102,9 +112,26 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( export interface AuthTransportLimits { requestTimeoutMs: number maxPendingRequests: number + maxResponseBytes: number } ``` +
+ +Interface AuthTransportLimits Details + +#### Property maxResponseBytes + +Maximum encoded application-response bytes retained for BRC-104 signing. +Set to `-1` only when the embedding service enforces an equivalent bound +before this middleware. Defaults to 8 MiB. + +```ts +maxResponseBytes: number +``` + +
+ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types) --- @@ -290,12 +317,10 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( Creates an Express middleware that handles authentication via BSV-SDK. ```ts -export function createAuthMiddleware( - options: AuthMiddlewareOptions -): (req: AuthRequest, res: Response, next: NextFunction) => void +export function createAuthMiddleware(options: AuthMiddlewareOptions): RequestHandler ``` -See also: [AuthMiddlewareOptions](#interface-authmiddlewareoptions), [AuthRequest](#interface-authrequest) +See also: [AuthMiddlewareOptions](#interface-authmiddlewareoptions)
diff --git a/packages/middleware/auth-express-middleware/README.md b/packages/middleware/auth-express-middleware/README.md index cc789bafb..9bb01495f 100644 --- a/packages/middleware/auth-express-middleware/README.md +++ b/packages/middleware/auth-express-middleware/README.md @@ -67,7 +67,8 @@ const auth = createAuthMiddleware({ logLevel: 'error', transportLimits: { requestTimeoutMs: 30_000, - maxPendingRequests: 1_000 + maxPendingRequests: 1_000, + maxResponseBytes: 8 * 1024 * 1024 } }) ``` @@ -87,6 +88,10 @@ const auth = createAuthMiddleware({ certificate, and response-signing state. It defaults to 30 seconds. - `transportLimits.maxPendingRequests` bounds per-process pending protocol state. It defaults to 1,000 and fails closed with `503` at capacity. +- `transportLimits.maxResponseBytes` bounds application responses buffered for + BRC-104 signing, including files passed to `res.sendFile`. It defaults to 8 + MiB and fails closed with a signed `413` response. Set it to `-1` only when + the embedding service enforces an equivalent response budget. Invalid option types fail during startup. @@ -191,7 +196,8 @@ messages: temporarily wrapped while an authenticated response is signed. - Do not trust `req.auth.identityKey === 'unknown'` as authorization. - Use shared session state for multi-instance deployments. -- Keep timeouts and capacity limits finite and monitor `408`/`503` rates. +- Keep timeouts, response sizes, and capacity limits finite and monitor + `408`/`413`/`503` rates. - Validate authorization separately after identity authentication. - Keep request body limits and normal Express hardening in place. diff --git a/packages/middleware/auth-express-middleware/package.json b/packages/middleware/auth-express-middleware/package.json index 7dc5d91e3..11d989111 100644 --- a/packages/middleware/auth-express-middleware/package.json +++ b/packages/middleware/auth-express-middleware/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/auth-express-middleware", - "version": "2.1.7", + "version": "2.2.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/middleware/auth-express-middleware/src/__tests/ExpressTransportHardening.test.ts b/packages/middleware/auth-express-middleware/src/__tests/ExpressTransportHardening.test.ts index 1c59f1bce..5b9446ae5 100644 --- a/packages/middleware/auth-express-middleware/src/__tests/ExpressTransportHardening.test.ts +++ b/packages/middleware/auth-express-middleware/src/__tests/ExpressTransportHardening.test.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { PrivateKey, Utils } from '@bsv/sdk' import { createAuthMiddleware, ExpressTransport } from '../index' import { MockWallet } from './MockWallet' @@ -80,6 +83,19 @@ function peerMock(overrides: Record = {}): any { } } +function peerWithSignedResponse(): { peer: any; signed: Promise } { + let resolveSigned!: () => void + const signed = new Promise(resolve => { + resolveSigned = resolve + }) + return { + peer: peerMock({ + toPeer: jest.fn().mockImplementation(async () => resolveSigned()) + }), + signed + } +} + function responsePayload( status: number, headers: Record, @@ -116,7 +132,10 @@ describe('ExpressTransport hardening', () => { [{ requestTimeoutMs: 0 }, 'requestTimeoutMs'], [{ requestTimeoutMs: 1.5 }, 'requestTimeoutMs'], [{ maxPendingRequests: 0 }, 'maxPendingRequests'], - [{ maxPendingRequests: Number.MAX_SAFE_INTEGER + 1 }, 'maxPendingRequests'] + [{ maxPendingRequests: Number.MAX_SAFE_INTEGER + 1 }, 'maxPendingRequests'], + [{ maxResponseBytes: 0 }, 'maxResponseBytes'], + [{ maxResponseBytes: -2 }, 'maxResponseBytes'], + [{ maxResponseBytes: Number.MAX_SAFE_INTEGER + 1 }, 'maxResponseBytes'] ])('rejects invalid transport limits', (limits, expected) => { expect(() => new ExpressTransport(false, undefined, undefined, limits)).toThrow(expected) }) @@ -536,6 +555,160 @@ describe('ExpressTransport hardening', () => { expect(transport.openGeneralHandles.has(REQUEST_ID)).toBe(false) }) + it('replaces an oversized authenticated response before signing', async () => { + const peer = peerMock() + const transport = new ExpressTransport(false, undefined, undefined, { + maxResponseBytes: 64 + }) + transport.peer = peer + const res = responseMock() + + ;(transport as any).setupAuthenticatedResponse( + validGeneralRequest(), + res, + jest.fn(), + IDENTITY_KEY, + REQUEST_ID + ) + + res.json({ value: 'x'.repeat(1_024) }) + await flushPromises() + + const expectedBody = Utils.toArray( + JSON.stringify({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }), + 'utf8' + ) + expect(peer.toPeer).toHaveBeenCalledWith(responsePayload(413, {}, expectedBody), SESSION_NONCE) + }) + + it.each([false, true])( + 'reads and signs an authenticated file within the response limit (unlimited: %s)', + async unlimited => { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'auth-express-file-')) + const filePath = join(temporaryDirectory, 'response.bin') + const contents = Buffer.from('bounded authenticated file') + writeFileSync(filePath, contents) + const { peer, signed } = peerWithSignedResponse() + const transport = new ExpressTransport(false, undefined, undefined, { + maxResponseBytes: unlimited ? -1 : contents.length + }) + transport.peer = peer + const res = responseMock() + + try { + ;(transport as any).setupAuthenticatedResponse( + validGeneralRequest(), + res, + jest.fn(), + IDENTITY_KEY, + REQUEST_ID + ) + res.sendFile(filePath) + await signed + + expect(peer.toPeer).toHaveBeenCalledWith( + responsePayload(200, {}, Array.from(contents)), + SESSION_NONCE + ) + } finally { + rmSync(temporaryDirectory, { recursive: true, force: true }) + } + } + ) + + it('stops an authenticated file read at the response limit and signs a 413', async () => { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'auth-express-file-')) + const filePath = join(temporaryDirectory, 'oversized.bin') + writeFileSync(filePath, Buffer.alloc(65, 7)) + const { peer, signed } = peerWithSignedResponse() + const transport = new ExpressTransport(false, undefined, undefined, { + maxResponseBytes: 64 + }) + transport.peer = peer + const res = responseMock() + + try { + ;(transport as any).setupAuthenticatedResponse( + validGeneralRequest(), + res, + jest.fn(), + IDENTITY_KEY, + REQUEST_ID + ) + res.sendFile(filePath) + await signed + + const expectedBody = Utils.toArray( + JSON.stringify({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }), + 'utf8' + ) + expect(peer.toPeer).toHaveBeenCalledWith( + responsePayload(413, {}, expectedBody), + SESSION_NONCE + ) + } finally { + rmSync(temporaryDirectory, { recursive: true, force: true }) + } + }) + + it('signs a 500 when an authenticated response file cannot be read', async () => { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'auth-express-file-')) + const filePath = join(temporaryDirectory, 'missing.bin') + const { peer, signed } = peerWithSignedResponse() + const transport = new ExpressTransport() + transport.peer = peer + const res = responseMock() + + try { + ;(transport as any).setupAuthenticatedResponse( + validGeneralRequest(), + res, + jest.fn(), + IDENTITY_KEY, + REQUEST_ID + ) + res.sendFile(filePath) + await signed + + expect(peer.toPeer).toHaveBeenCalledWith(responsePayload(500, {}), SESSION_NONCE) + } finally { + rmSync(temporaryDirectory, { recursive: true, force: true }) + } + }) + + it('forwards authenticated response file errors to the sendFile callback overload', async () => { + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'auth-express-file-')) + const filePath = join(temporaryDirectory, 'missing.bin') + const transport = new ExpressTransport() + const peer = peerMock() + transport.peer = peer + const res = responseMock() + + try { + ;(transport as any).setupAuthenticatedResponse( + validGeneralRequest(), + res, + jest.fn(), + IDENTITY_KEY, + REQUEST_ID + ) + const callbackError = new Promise(resolve => res.sendFile(filePath, resolve)) + + await expect(callbackError).resolves.toHaveProperty('name', 'Error') + expect(peer.toPeer).not.toHaveBeenCalled() + } finally { + rmSync(temporaryDirectory, { recursive: true, force: true }) + } + }) + it('buffers and signs an authenticated text response with its inferred content type', async () => { const debug = jest.fn() const logger = { diff --git a/packages/middleware/auth-express-middleware/src/index.ts b/packages/middleware/auth-express-middleware/src/index.ts index 8258a0cb4..4cc246b25 100644 --- a/packages/middleware/auth-express-middleware/src/index.ts +++ b/packages/middleware/auth-express-middleware/src/index.ts @@ -35,6 +35,7 @@ export { writeBodyToWriter } from './authMiddlewareHelpers.js' const WELL_KNOWN_AUTH_PATH = '/.well-known/auth' const DEFAULT_REQUEST_TIMEOUT_MS = 30_000 const DEFAULT_MAX_PENDING_REQUESTS = 1_000 +const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024 const MAX_AUTH_HEADER_LENGTH = 4_096 const TRACEPARENT_PATTERN = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/i @@ -72,6 +73,12 @@ interface ActiveCertificateRequest { export interface AuthTransportLimits { requestTimeoutMs: number maxPendingRequests: number + /** + * Maximum encoded application-response bytes retained for BRC-104 signing. + * Set to `-1` only when the embedding service enforces an equivalent bound + * before this middleware. Defaults to 8 MiB. + */ + maxResponseBytes: number } export interface AuthRequest extends Request { @@ -239,6 +246,37 @@ function safeErrorDetails(error: unknown): Record { return error instanceof Error ? { errorName: error.name } : { errorType: typeof error } } +class ResponseFileTooLargeError extends Error { + constructor() { + super('The response file exceeds the configured service limit.') + this.name = 'ResponseFileTooLargeError' + } +} + +type BoundedFileReadResult = { ok: true; data: Buffer } | { ok: false; error: Error } + +function readFileWithinLimit( + path: string, + maxBytes: number, + callback: (result: BoundedFileReadResult) => void +): void { + const stream = fs.createReadStream(path) + const chunks: Buffer[] = [] + let totalBytes = 0 + + stream.on('data', (buffer: Buffer) => { + if (maxBytes !== -1 && totalBytes + buffer.length > maxBytes) { + stream.destroy() + callback({ ok: false, error: new ResponseFileTooLargeError() }) + return + } + totalBytes += buffer.length + chunks.push(buffer) + }) + stream.once('error', error => callback({ ok: false, error })) + stream.once('end', () => callback({ ok: true, data: Buffer.concat(chunks, totalBytes) })) +} + /** * ResponseWriterWrapper buffers response data until signing is complete. * This pattern matches the Go implementation for cleaner response handling. @@ -248,6 +286,8 @@ class ResponseWriterWrapper { private headers: Record = {} private body: number[] = [] + constructor(private readonly maxResponseBytes: number) {} + status(code: number): this { this.statusCode = code return this @@ -265,7 +305,7 @@ class ResponseWriterWrapper { } send(data: any): this { - this.body = convertValueToArray(data, this.headers) + this.setBody(convertValueToArray(data, this.headers)) return this } @@ -273,7 +313,7 @@ class ResponseWriterWrapper { if (!this.headers['content-type']) { this.headers['content-type'] = 'application/json' } - this.body = Utils.toArray(JSON.stringify(data), 'utf8') + this.setBody(Utils.toArray(JSON.stringify(data), 'utf8')) return this } @@ -281,7 +321,7 @@ class ResponseWriterWrapper { if (!this.headers['content-type']) { this.headers['content-type'] = 'text/plain' } - this.body = Utils.toArray(data, 'utf8') + this.setBody(Utils.toArray(data, 'utf8')) return this } @@ -301,6 +341,36 @@ class ResponseWriterWrapper { getBody(): number[] { return this.body } + + exceedsLimit(byteLength: number): boolean { + return this.maxResponseBytes !== -1 && byteLength > this.maxResponseBytes + } + + rejectTooLarge(): void { + this.setTooLargeError() + } + + private setBody(body: number[]): void { + if (!this.exceedsLimit(body.length)) { + this.body = body + return + } + + this.setTooLargeError() + } + + private setTooLargeError(): void { + this.statusCode = 413 + this.headers['content-type'] = 'application/json' + this.body = Utils.toArray( + JSON.stringify({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }), + 'utf8' + ) + } } /** @@ -345,16 +415,23 @@ export class ExpressTransport implements Transport { } const requestTimeoutMs = limits.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS const maxPendingRequests = limits.maxPendingRequests ?? DEFAULT_MAX_PENDING_REQUESTS + const maxResponseBytes = limits.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES if (!Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 1) { throw new RangeError('requestTimeoutMs must be a positive safe integer.') } if (!Number.isSafeInteger(maxPendingRequests) || maxPendingRequests < 1) { throw new RangeError('maxPendingRequests must be a positive safe integer.') } + if ( + !Number.isSafeInteger(maxResponseBytes) || + (maxResponseBytes !== -1 && maxResponseBytes < 1) + ) { + throw new RangeError('maxResponseBytes must be -1 or a positive safe integer.') + } this.allowUnauthenticated = allowUnauthenticated this.logger = logger this.logLevel = logLevel || 'error' // Default to 'error' if not provided - this.limits = { requestTimeoutMs, maxPendingRequests } + this.limits = { requestTimeoutMs, maxPendingRequests, maxResponseBytes } } /** @@ -959,7 +1036,7 @@ export class ExpressTransport implements Transport { req.auth = { identityKey: senderPublicKey } const sessionNonce = singleHeader(req, 'x-bsv-auth-your-nonce') as string - const wrapper = new ResponseWriterWrapper() + const wrapper = new ResponseWriterWrapper(this.limits.maxResponseBytes) let responseSent = false const buildAndSendResponse = async (): Promise => { @@ -1075,11 +1152,26 @@ export class ExpressTransport implements Transport { } ;(res as any).__sendFile = res.sendFile - ;(res as any).sendFile = (path: string, options?: any, callback?: Function) => { - fs.readFile(path, (err, data) => { - if (err) { - this.log('error', 'Error reading file in sendFile', { errorName: err.name }) - if (callback) return callback(err) + ;(res as any).sendFile = ( + path: string, + optionsOrCallback?: unknown, + callback?: (error: Error) => void + ) => { + const errorCallback = + typeof optionsOrCallback === 'function' + ? (optionsOrCallback as (error: Error) => void) + : callback + readFileWithinLimit(path, this.limits.maxResponseBytes, result => { + if (!result.ok) { + if (result.error instanceof ResponseFileTooLargeError) { + wrapper.rejectTooLarge() + buildAndSendResponse() + return + } + this.log('error', 'Error reading file in sendFile', { + errorName: result.error.name + }) + if (errorCallback != null) return errorCallback(result.error) wrapper.status(500) buildAndSendResponse() return @@ -1087,7 +1179,7 @@ export class ExpressTransport implements Transport { const mimeType = mime.lookup(path) || 'application/octet-stream' wrapper.set('Content-Type', mimeType) - wrapper.send(Array.from(data)) + wrapper.send(result.data) buildAndSendResponse() }) } diff --git a/packages/overlays/overlay-express/package.json b/packages/overlays/overlay-express/package.json index 71bb97f58..5aaa108ad 100644 --- a/packages/overlays/overlay-express/package.json +++ b/packages/overlays/overlay-express/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-express", - "version": "2.4.9", + "version": "2.5.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay-express/src/BanService.ts b/packages/overlays/overlay-express/src/BanService.ts index 2db07317b..ff5a6eedb 100644 --- a/packages/overlays/overlay-express/src/BanService.ts +++ b/packages/overlays/overlay-express/src/BanService.ts @@ -126,12 +126,23 @@ export class BanService { return record !== null } - /** - * Lists all bans, optionally filtered by type. - */ - async listBans (type?: 'domain' | 'outpoint'): Promise { + /** Lists bans in newest-first order using an operator-bounded page. */ + async listBans (type?: 'domain' | 'outpoint', limit: number = 1000, skip: number = 0): Promise { + if (limit !== -1 && (!Number.isSafeInteger(limit) || limit < 1)) { + throw new TypeError('limit must be a positive integer or -1') + } + if (!Number.isSafeInteger(skip) || skip < 0) { + throw new TypeError('skip must be a non-negative integer') + } const query = typeof type === 'string' ? { type } : {} - return await this.bans.find(query).sort({ bannedAt: -1 }).toArray() + let cursor: any = this.bans.find(query).sort({ bannedAt: -1 }) + if (skip > 0 && typeof cursor.skip === 'function') cursor = cursor.skip(skip) + if (limit !== -1 && typeof cursor.limit === 'function') cursor = cursor.limit(limit) + const records = await cursor.toArray() + // MongoDB always provides skip/limit. The slice also keeps lightweight + // Mongo-compatible adapters and test doubles within the same contract. + const skipped = typeof cursor.skip === 'function' ? records : records.slice(skip) + return limit === -1 || typeof cursor.limit === 'function' ? skipped : skipped.slice(0, limit) } /** diff --git a/packages/overlays/overlay-express/src/JanitorService.ts b/packages/overlays/overlay-express/src/JanitorService.ts index 2ea117346..c305c5db9 100644 --- a/packages/overlays/overlay-express/src/JanitorService.ts +++ b/packages/overlays/overlay-express/src/JanitorService.ts @@ -20,6 +20,10 @@ export interface JanitorConfig { * development environments. Never enable this on an internet-facing node. */ allowPrivateHosts?: boolean + /** Mongo cursor batch size. This controls retained scan memory, not coverage. */ + batchSize?: number + /** Maximum detailed results retained in a report. Use -1 to retain all results. */ + maxReportResults?: number } /** @@ -49,6 +53,7 @@ export interface JanitorReport { durationMs: number shipResults: HostHealthResult[] slapResults: HostHealthResult[] + resultsTruncated: boolean summary: { totalChecked: number healthy: number @@ -74,6 +79,8 @@ export class JanitorService { private readonly banService?: BanService private readonly autoBanOnRemoval: boolean private readonly allowPrivateHosts: boolean + private readonly batchSize: number + private readonly maxReportResults: number constructor (config: JanitorConfig) { this.mongoDb = config.mongoDb @@ -83,6 +90,14 @@ export class JanitorService { this.banService = config.banService this.autoBanOnRemoval = config.autoBanOnRemoval ?? true this.allowPrivateHosts = config.allowPrivateHosts ?? false + this.batchSize = config.batchSize ?? 250 + this.maxReportResults = config.maxReportResults ?? 1000 + if (this.batchSize !== -1 && (!Number.isSafeInteger(this.batchSize) || this.batchSize < 1)) { + throw new TypeError('Janitor batchSize must be a positive integer or -1') + } + if (this.maxReportResults !== -1 && (!Number.isSafeInteger(this.maxReportResults) || this.maxReportResults < 1)) { + throw new TypeError('Janitor maxReportResults must be a positive integer or -1') + } } /** @@ -97,17 +112,26 @@ export class JanitorService { let slapResults: HostHealthResult[] = [] let removed = 0 let banned = 0 + let totalChecked = 0 + let healthy = 0 + let unhealthy = 0 try { const shipCheckResult = await this.checkTopicOutputs('shipRecords', 'topic') shipResults = shipCheckResult.results removed += shipCheckResult.removed banned += shipCheckResult.banned + totalChecked += shipCheckResult.checked + healthy += shipCheckResult.healthy + unhealthy += shipCheckResult.unhealthy const slapCheckResult = await this.checkTopicOutputs('slapRecords', 'service') slapResults = slapCheckResult.results removed += slapCheckResult.removed banned += slapCheckResult.banned + totalChecked += slapCheckResult.checked + healthy += slapCheckResult.healthy + unhealthy += slapCheckResult.unhealthy this.logger.log(chalk.green('Janitor health checks completed')) } catch (error) { @@ -116,17 +140,17 @@ export class JanitorService { } const completedAt = new Date() - const allResults = [...shipResults, ...slapResults] return { startedAt, completedAt, durationMs: completedAt.getTime() - startedAt.getTime(), shipResults, slapResults, + resultsTruncated: totalChecked > shipResults.length + slapResults.length, summary: { - totalChecked: allResults.length, - healthy: allResults.filter(r => r.healthy).length, - unhealthy: allResults.filter(r => !r.healthy).length, + totalChecked, + healthy, + unhealthy, removed, banned } @@ -205,9 +229,18 @@ export class JanitorService { const shipCollection = this.mongoDb.collection('shipRecords') const slapCollection = this.mongoDb.collection('slapRecords') + const readStatusRecords = async (collection: any): Promise[]> => { + let cursor = collection.find({}) + if (this.maxReportResults !== -1 && typeof cursor.limit === 'function') { + cursor = cursor.limit(this.maxReportResults) + } + const records = await cursor.toArray() + return this.maxReportResults === -1 ? records : records.slice(0, this.maxReportResults) + } + const [shipOutputs, slapOutputs] = await Promise.all([ - shipCollection.find({}).toArray(), - slapCollection.find({}).toArray() + readStatusRecords(shipCollection), + readStatusRecords(slapCollection) ]) const shipResults: HostHealthResult[] = shipOutputs.map(output => ({ @@ -241,25 +274,53 @@ export class JanitorService { private async checkTopicOutputs ( collectionName: string, typeField: 'topic' | 'service' - ): Promise<{ results: HostHealthResult[], removed: number, banned: number }> { + ): Promise<{ + results: HostHealthResult[] + checked: number + healthy: number + unhealthy: number + removed: number + banned: number + }> { const results: HostHealthResult[] = [] + let checked = 0 + let healthy = 0 + let unhealthy = 0 let removed = 0 let banned = 0 try { const collection = this.mongoDb.collection(collectionName) - const outputs = await collection.find({}).toArray() + let cursor: any = collection.find({}) + if (this.batchSize !== -1 && typeof cursor.batchSize === 'function') { + cursor = cursor.batchSize(this.batchSize) + } - this.logger.log(chalk.cyan(`Checking ${outputs.length} ${collectionName} outputs...`)) + this.logger.log(chalk.cyan(`Checking ${collectionName} outputs in bounded batches...`)) - for (const output of outputs) { + const processOutput = async (output: Record): Promise => { const result = await this.checkOutput(output, collection, typeField) - results.push(result) + checked++ + if (result.healthy) healthy++ + else unhealthy++ + if (this.maxReportResults === -1 || results.length < this.maxReportResults) { + results.push(result) + } if (result.error === 'REMOVED') { removed++ } } + if (cursor?.[Symbol.asyncIterator] !== undefined) { + for await (const output of cursor as AsyncIterable>) { + await processOutput(output) + } + } else { + // Compatibility for lightweight Mongo-compatible adapters and test doubles. + const outputs = await cursor.toArray() + for (const output of outputs) await processOutput(output) + } + // Count auto-bans that happened during this run if (this.banService !== undefined && this.autoBanOnRemoval) { banned = removed // Each removal triggers a ban @@ -268,7 +329,7 @@ export class JanitorService { this.logger.error(chalk.red(`Error checking ${collectionName} outputs:`), error) } - return { results, removed, banned } + return { results, checked, healthy, unhealthy, removed, banned } } /** diff --git a/packages/overlays/overlay-express/src/OverlayExpress.ts b/packages/overlays/overlay-express/src/OverlayExpress.ts index ab159b452..5278d8a5f 100644 --- a/packages/overlays/overlay-express/src/OverlayExpress.ts +++ b/packages/overlays/overlay-express/src/OverlayExpress.ts @@ -41,6 +41,7 @@ import { createHash, timingSafeEqual } from 'node:crypto' import { JanitorService, type JanitorReport } from './JanitorService.js' import { BanService } from './BanService.js' import { BanAwareLookupWrapper } from './BanAwareLookupWrapper.js' +import { ResourceBoundedLookupWrapper } from './ResourceBoundedLookupWrapper.js' import { BanAwareTopicManager } from './BanAwareTopicManager.js' import { BanAwareSHIPStorage, BanAwareSLAPStorage } from './BanAwareDiscoveryStorage.js' import { ReorgSseAdapter, type ReorgHandlerInput } from './ReorgStream.js' @@ -55,7 +56,12 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders, type HttpServerPolicyDefaults, type SecurityHeadersOptions @@ -75,14 +81,14 @@ interface Migration { * Allows running migrations defined in code rather than files. */ class InMemoryMigrationSource implements Knex.Knex.MigrationSource { - constructor (private readonly migrations: Migration[]) { } + constructor(private readonly migrations: Migration[]) {} /** * Gets the list of migrations. * @param loadExtensions - Array of file extensions to filter by (not used here) * @returns Promise resolving to the array of migrations */ - async getMigrations (_loadExtensions: readonly string[]): Promise { + async getMigrations(_loadExtensions: readonly string[]): Promise { return this.migrations } @@ -91,8 +97,10 @@ class InMemoryMigrationSource implements Knex.Knex.MigrationSource { * @param migration - The migration object * @returns The name of the migration */ - getMigrationName (migration: Migration): string { - return typeof migration.name === 'string' ? migration.name : `Migration at index ${this.migrations.indexOf(migration)}` + getMigrationName(migration: Migration): string { + return typeof migration.name === 'string' + ? migration.name + : `Migration at index ${this.migrations.indexOf(migration)}` } /** @@ -100,7 +108,7 @@ class InMemoryMigrationSource implements Knex.Knex.MigrationSource { * @param migration - The migration object * @returns Promise resolving to the migration object */ - async getMigration (migration: Migration): Promise { + async getMigration(migration: Migration): Promise { return await Promise.resolve(migration) } } @@ -129,6 +137,8 @@ export interface EngineConfig { reorgStreamUrl?: string reorgScanDepth?: number unprovenMaintenanceIntervalMs?: number + /** Maximum lookup formulas hydrated by the engine. Use -1 to opt out. */ + maxLookupResults?: number } export type HealthStatus = 'ok' | 'degraded' | 'error' @@ -143,7 +153,10 @@ export interface HealthCheckResult { durationMs: number } -export type HealthCheckHandler = () => Promise | void> | Omit | void +export type HealthCheckHandler = () => + | Promise | void> + | Omit + | void export interface HealthCheckDefinition { name: string @@ -186,11 +199,14 @@ export interface EdgePolicyConfig { securityHeaders: SecurityHeadersOptions } -export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise<{ - blockHeight: number - blockHash: string - merkleRoot?: string -} | undefined> +export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise< + | { + blockHeight: number + blockHash: string + merkleRoot?: string + } + | undefined +> interface BASMCapableEngine extends Engine { provideTopicAnchorTip: (topic: string) => Promise @@ -200,56 +216,67 @@ interface BASMCapableEngine extends Engine { provideRawTransactions: (txids: string[]) => Promise startBASMSync: () => Promise advanceTopicAnchorChains: (toHeight?: number) => Promise - evictUnprovenTransactions: (options?: { topic?: string, thresholdBlocks?: number }) => Promise + evictUnprovenTransactions: (options?: { + topic?: string + thresholdBlocks?: number + }) => Promise refreshUnprovenTransactionProofs: (options: { topic?: string thresholdBlocks?: number - proofProvider: (txid: string) => Promise<{ merklePath: MerklePath, blockHeight?: number } | undefined> + proofProvider: ( + txid: string + ) => Promise<{ merklePath: MerklePath; blockHeight?: number } | undefined> }) => Promise maintainUnprovenTransactions: (options: { topic?: string thresholdBlocks?: number - proofProvider: (txid: string) => Promise<{ merklePath: MerklePath, blockHeight?: number } | undefined> + proofProvider: ( + txid: string + ) => Promise<{ merklePath: MerklePath; blockHeight?: number } | undefined> }) => Promise - evictAppliedTransaction: (txid: string, options?: { topic?: string, reason?: string }) => Promise + evictAppliedTransaction: ( + txid: string, + options?: { topic?: string; reason?: string } + ) => Promise handleReorg: (input: ReorgHandlerInput) => Promise revalidateRecentAnchors: (depth?: number) => Promise } class PublicRequestError extends Error { - constructor (message: string) { + constructor(message: string) { super(message) this.name = 'PublicRequestError' } } -function publicErrorMessage ( +function publicErrorMessage( error: unknown, fallback: string = 'Request could not be processed' ): string { return error instanceof PublicRequestError ? error.message : fallback } -function secretMatches (provided: string, expected: string): boolean { +function secretMatches(provided: string, expected: string): boolean { const providedDigest = createHash('sha256').update(provided, 'utf8').digest() const expectedDigest = createHash('sha256').update(expected, 'utf8').digest() return timingSafeEqual(providedDigest, expectedDigest) } -function parseTopicsHeader (header: string): string[] { +function parseTopicsHeader(header: string): string[] { const value = header.trim() let parsed: unknown try { - parsed = value.startsWith('[') - ? JSON.parse(value) - : value.split(',').map(topic => topic.trim()) + parsed = value.startsWith('[') ? JSON.parse(value) : value.split(',').map(topic => topic.trim()) } catch { throw new PublicRequestError( 'Invalid x-topics header: expected a comma-separated list or JSON string array' ) } - if (!Array.isArray(parsed) || parsed.some(topic => typeof topic !== 'string' || topic.length === 0)) { + if ( + !Array.isArray(parsed) || + parsed.some(topic => typeof topic !== 'string' || topic.length === 0) + ) { throw new PublicRequestError( 'Invalid x-topics header: expected a comma-separated list or JSON string array' ) @@ -374,12 +401,16 @@ export default class OverlayExpress { hostDownRevokeScore: number autoBanOnRemoval: boolean allowPrivateHosts: boolean + batchSize?: number + maxReportResults?: number } = { - requestTimeoutMs: 10000, // 10 seconds - hostDownRevokeScore: 3, - autoBanOnRemoval: true, - allowPrivateHosts: false - } + requestTimeoutMs: 10000, // 10 seconds + hostDownRevokeScore: 3, + autoBanOnRemoval: true, + allowPrivateHosts: false, + batchSize: undefined, + maxReportResults: undefined + } // Ban service for persistent domain/outpoint blocking banService?: BanService @@ -426,7 +457,8 @@ export default class OverlayExpress { maxRequestsPerSocket: 1_000 }, securityHeaders: { - contentSecurityPolicy: "default-src 'none'; script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; img-src 'self' data: https://bsvblockchain.org; connect-src 'self' https:; font-src 'self' https://cdn.jsdelivr.net; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + contentSecurityPolicy: + "default-src 'none'; script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; img-src 'self' data: https://bsvblockchain.org; connect-src 'self' https:; font-src 'self' https://cdn.jsdelivr.net; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" } } @@ -438,7 +470,7 @@ export default class OverlayExpress { * @param adminToken - Optional. An administrative Bearer token used to protect admin routes. * If not provided, a random token will be generated at runtime. */ - constructor ( + constructor( public name: string, public privateKey: string, public advertisableFQDN: string, @@ -452,7 +484,7 @@ export default class OverlayExpress { /** * Returns the current admin token in case you need to programmatically retrieve or display it. */ - getAdminToken (): string { + getAdminToken(): string { return this.adminToken } @@ -460,7 +492,7 @@ export default class OverlayExpress { * Configures the port on which the server will listen. * @param port - The port number */ - configurePort (port: number): void { + configurePort(port: number): void { this.port = port this.logger.log(chalk.blue(`Server port set to ${port}`)) } @@ -469,7 +501,7 @@ export default class OverlayExpress { * Configures the web user interface * @param config - Web UI configuration options */ - configureWebUI (config: UIConfig): void { + configureWebUI(config: UIConfig): void { this.webUIConfig = config this.logger.log(chalk.blue('Web UI has been configured.')) } @@ -481,8 +513,10 @@ export default class OverlayExpress { * - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3) * - autoBanOnRemoval: Whether to auto-ban domains when removed by janitor (default: true) * - allowPrivateHosts: Permit private/HTTP targets for isolated local development (default: false) + * - batchSize: Mongo scan batch size; -1 uses the driver default + * - maxReportResults: Maximum retained result details; -1 retains all */ - configureJanitor (config: Partial): void { + configureJanitor(config: Partial): void { this.janitorConfig = { ...this.janitorConfig, ...config @@ -493,7 +527,7 @@ export default class OverlayExpress { /** * Configures health-report behavior. */ - configureHealth (config: Partial): void { + configureHealth(config: Partial): void { this.healthConfig = { ...this.healthConfig, ...config @@ -506,10 +540,12 @@ export default class OverlayExpress { * Public cross-origin access is the default; pass allowedOrigins or set * OVERLAY_CORS_MODE=allowlist to restrict browser callers. */ - configureEdgePolicy (config: Partial> & { - http?: Partial - securityHeaders?: SecurityHeadersOptions - }): void { + configureEdgePolicy( + config: Partial> & { + http?: Partial + securityHeaders?: SecurityHeadersOptions + } + ): void { const { http, securityHeaders: headerConfig, ...topLevel } = config const definedTopLevel = Object.fromEntries( Object.entries(topLevel).filter(([, value]) => value !== undefined) @@ -538,7 +574,7 @@ export default class OverlayExpress { /** * Registers an application-specific health check. */ - registerHealthCheck (definition: HealthCheckDefinition): void { + registerHealthCheck(definition: HealthCheckDefinition): void { this.healthChecks = this.healthChecks.filter(check => check.name !== definition.name) this.healthChecks.push({ scope: 'ready', @@ -555,7 +591,7 @@ export default class OverlayExpress { * * @param identityKey - The hex-encoded public key of the admin */ - configureAdminIdentityKey (identityKey: string): void { + configureAdminIdentityKey(identityKey: string): void { this.adminIdentityKey = identityKey this.logger.log(chalk.blue('Admin identity key has been configured.')) } @@ -565,7 +601,7 @@ export default class OverlayExpress { * middleware. Horizontally scaled services should supply a shared * AsyncSessionManager rather than use the default process-local store. */ - configureAuthSessionManager (sessionManager: SessionManager | AsyncSessionManager): void { + configureAuthSessionManager(sessionManager: SessionManager | AsyncSessionManager): void { this.authSessionManager = sessionManager this.logger.log(chalk.blue('BSV authentication session manager has been configured.')) } @@ -574,7 +610,7 @@ export default class OverlayExpress { * Configures the logger to be used by the server. * @param logger - A logger object (e.g., console) */ - configureLogger (logger: typeof console): void { + configureLogger(logger: typeof console): void { this.logger = logger this.logger.log(chalk.blue('Logger has been configured.')) } @@ -584,7 +620,7 @@ export default class OverlayExpress { * By default, it re-initializes chainTracker as a WhatsOnChain for that network. * @param network - The network ('main' or 'test') */ - configureNetwork (network: 'main' | 'test'): void { + configureNetwork(network: 'main' | 'test'): void { this.network = network this.chainTracker = new WhatsOnChain(this.network) this.logger.log(chalk.blue(`Network set to ${network}`)) @@ -595,7 +631,9 @@ export default class OverlayExpress { * If 'scripts only' is used, it implies no full SPV chain tracking in the Engine. * @param chainTracker - An instance of ChainTracker or 'scripts only' */ - configureChainTracker (chainTracker: ChainTracker | 'scripts only' = new WhatsOnChain(this.network)): void { + configureChainTracker( + chainTracker: ChainTracker | 'scripts only' = new WhatsOnChain(this.network) + ): void { this.chainTracker = chainTracker this.logger.log(chalk.blue('ChainTracker has been configured.')) } @@ -604,7 +642,7 @@ export default class OverlayExpress { * Configures the ARC API key. * @param apiKey - The ARC API key */ - configureArcApiKey (apiKey: string): void { + configureArcApiKey(apiKey: string): void { this.arcApiKey = apiKey this.logger.log(chalk.blue('ARC API key has been configured.')) } @@ -613,7 +651,7 @@ export default class OverlayExpress { * Configures the ARC callback token expected by /arc-ingest. * @param token - The token ARC should present when posting callback notifications. */ - configureArcCallbackToken (token: string): void { + configureArcCallbackToken(token: string): void { this.arcCallbackToken = token this.logger.log(chalk.blue('ARC callback token has been configured.')) } @@ -621,11 +659,14 @@ export default class OverlayExpress { /** * Configures Arcade for first-choice transaction propagation and proof lookup. */ - configureArcade (url: string, config: { - apiKey?: string - deploymentId?: string - chaintracksApiPrefix?: string - } = {}): void { + configureArcade( + url: string, + config: { + apiKey?: string + deploymentId?: string + chaintracksApiPrefix?: string + } = {} + ): void { this.arcadeUrl = url this.arcadeApiKey = config.apiKey this.arcadeDeploymentId = config.deploymentId @@ -639,11 +680,14 @@ export default class OverlayExpress { * Configures a go-chaintracks compatible service for header validation and * BASM reorg streaming. Arcade exposes this at `/chaintracks/v2`. */ - configureChaintracks (url: string, config: { - apiPrefix?: string - reorgStream?: boolean - scanDepth?: number - } = {}): void { + configureChaintracks( + url: string, + config: { + apiPrefix?: string + reorgStream?: boolean + scanDepth?: number + } = {} + ): void { const apiPrefix = config.apiPrefix ?? '/chaintracks/v2' const client = new ChaintracksProvider(url, { apiPrefix }) this.configureChainTracker(client) @@ -667,7 +711,7 @@ export default class OverlayExpress { * This is a broad toggle that can be overridden or customized through syncConfiguration. * @param enable - true to enable, false to disable */ - configureEnableGASPSync (enable: boolean): void { + configureEnableGASPSync(enable: boolean): void { this.enableGASPSync = enable this.logger.log(chalk.blue(`GASP synchronization ${enable ? 'enabled' : 'disabled'}.`)) } @@ -676,7 +720,7 @@ export default class OverlayExpress { * Enables or disables BRC-136 BASM synchronization. * BASM is opt-in because it requires direct proofs and block hash resolution. */ - configureEnableBASMSync (enable: boolean): void { + configureEnableBASMSync(enable: boolean): void { this.enableBASMSync = enable this.logger.log(chalk.blue(`BASM synchronization ${enable ? 'enabled' : 'disabled'}.`)) } @@ -684,7 +728,7 @@ export default class OverlayExpress { /** * Configures the block header resolver used to derive BASM block hashes. */ - configureTopicAnchorHeaderResolver (resolver: TopicAnchorHeaderResolver): void { + configureTopicAnchorHeaderResolver(resolver: TopicAnchorHeaderResolver): void { this.topicAnchorHeaderResolver = resolver this.logger.log(chalk.blue('BASM topic anchor header resolver has been configured.')) } @@ -695,7 +739,7 @@ export default class OverlayExpress { * @param url - The reorg stream URL, e.g. `https://arcade.example/v2/reorg/stream`. * @param scanDepth - Optional revalidation-sweep depth in blocks (default 3). */ - configureReorgStream (url: string, scanDepth?: number): void { + configureReorgStream(url: string, scanDepth?: number): void { this.reorgStreamUrl = url if (scanDepth !== undefined) { this.reorgScanDepth = scanDepth @@ -706,7 +750,7 @@ export default class OverlayExpress { /** * Configures the opt-in unproven state eviction threshold. */ - configureUnprovenEviction (config: { thresholdBlocks?: number }): void { + configureUnprovenEviction(config: { thresholdBlocks?: number }): void { if (config.thresholdBlocks !== undefined) { this.unprovenEvictionBlocks = config.thresholdBlocks } @@ -717,7 +761,7 @@ export default class OverlayExpress { * Configures periodic unproven maintenance. Each run first tries configured * proof providers, then evicts rows that are still unproven past the threshold. */ - configureUnprovenMaintenance (config: { intervalMs?: number, thresholdBlocks?: number }): void { + configureUnprovenMaintenance(config: { intervalMs?: number; thresholdBlocks?: number }): void { if (config.intervalMs !== undefined) { this.unprovenMaintenanceIntervalMs = config.intervalMs } @@ -731,7 +775,7 @@ export default class OverlayExpress { * Configures how often the BASM anchor chain is extended with empty anchors to * follow the chain tip. Set to 0 to disable periodic polling. */ - configureBASMBlockPollInterval (intervalMs: number): void { + configureBASMBlockPollInterval(intervalMs: number): void { this.basmBlockPollIntervalMs = intervalMs this.logger.log(chalk.blue(`BASM block poll interval set to ${intervalMs}ms.`)) } @@ -740,7 +784,7 @@ export default class OverlayExpress { * Enables or disables verbose request logging. * @param enable - true to enable, false to disable */ - configureVerboseRequestLogging (enable: boolean): void { + configureVerboseRequestLogging(enable: boolean): void { this.verboseRequestLogging = enable this.logger.log(chalk.blue(`Verbose request logging ${enable ? 'enabled' : 'disabled'}.`)) } @@ -749,7 +793,7 @@ export default class OverlayExpress { * Configure Knex (SQL) database connection. * @param config - Knex configuration object, or a MySQL connection string loaded from configuration. */ - async configureKnex (config: Knex.Knex.Config | string): Promise { + async configureKnex(config: Knex.Knex.Config | string): Promise { if (typeof config === 'string') { config = { client: 'mysql2', @@ -765,7 +809,7 @@ export default class OverlayExpress { * Also initializes the BanService for persistent ban tracking. * @param connectionString - MongoDB connection string */ - async configureMongo (connectionString: string): Promise { + async configureMongo(connectionString: string): Promise { const mongoClient = new MongoClient(connectionString) await mongoClient.connect() this.mongoClient = mongoClient @@ -784,7 +828,7 @@ export default class OverlayExpress { * @param name - The name of the Topic Manager * @param manager - An instance of TopicManager */ - configureTopicManager (name: string, manager: TopicManager): void { + configureTopicManager(name: string, manager: TopicManager): void { this.managers[name] = manager this.logger.log(chalk.blue(`Configured topic manager ${name}`)) } @@ -794,7 +838,7 @@ export default class OverlayExpress { * @param name - The name of the Lookup Service * @param service - An instance of LookupService */ - configureLookupService (name: string, service: LookupService): void { + configureLookupService(name: string, service: LookupService): void { this.services[name] = service this.logger.log(chalk.blue(`Configured lookup service ${name}`)) } @@ -804,9 +848,9 @@ export default class OverlayExpress { * @param name - The name of the Lookup Service * @param serviceFactory - A factory function that creates a LookupService instance using Knex */ - configureLookupServiceWithKnex ( + configureLookupServiceWithKnex( name: string, - serviceFactory: (knex: Knex.Knex) => { service: LookupService, migrations: Migration[] } + serviceFactory: (knex: Knex.Knex) => { service: LookupService; migrations: Migration[] } ): void { const knex = this.ensureKnex() const factoryResult = serviceFactory(knex) @@ -820,7 +864,10 @@ export default class OverlayExpress { * @param name - The name of the Lookup Service * @param serviceFactory - A factory function that creates a LookupService instance using MongoDB */ - configureLookupServiceWithMongo (name: string, serviceFactory: (mongoDb: Db) => LookupService): void { + configureLookupServiceWithMongo( + name: string, + serviceFactory: (mongoDb: Db) => LookupService + ): void { const mongoDb = this.ensureMongo() this.services[name] = serviceFactory(mongoDb) this.logger.log(chalk.blue(`Configured lookup service ${name} with MongoDB`)) @@ -840,7 +887,7 @@ export default class OverlayExpress { * These fields will be respected when we finally build/configure the Engine * in the `configureEngine()` method below. */ - configureEngineParams (params: EngineConfig): void { + configureEngineParams(params: EngineConfig): void { this.engineConfig = { ...this.engineConfig, ...params @@ -859,8 +906,20 @@ export default class OverlayExpress { * * @param autoConfigureShipSlap - Whether to auto-configure SHIP and SLAP services (default: true) */ - async configureEngine (autoConfigureShipSlap = true): Promise { + async configureEngine(autoConfigureShipSlap = true): Promise { const knex = this.ensureKnex() + const maxLookupResults = + this.engineConfig.maxLookupResults ?? + readResourceLimit( + this.edgePolicyConfig.environmentPrefix, + 'MAX_LOOKUP_RESULTS', + profileValue(readResourceProfile(this.edgePolicyConfig.environmentPrefix), { + small: 500, + standard: 1000, + highThroughput: 5000 + }), + 1_000_000 + ) if (autoConfigureShipSlap) { const mongoDb = this.ensureMongo() @@ -875,15 +934,23 @@ export default class OverlayExpress { this.configureTopicManager('tm_ship', new DiscoveryServices.SHIPTopicManager()) this.configureTopicManager('tm_slap', new DiscoveryServices.SLAPTopicManager()) - const shipStorageForLookup = this.banService === undefined - ? shipStorage - : new BanAwareSHIPStorage(shipStorage, this.banService, this.logger) - const slapStorageForLookup = this.banService === undefined - ? slapStorage - : new BanAwareSLAPStorage(slapStorage, this.banService, this.logger) - - this.services.ls_ship = new DiscoveryServices.SHIPLookupService(shipStorageForLookup as any) - this.services.ls_slap = new DiscoveryServices.SLAPLookupService(slapStorageForLookup as any) + const shipStorageForLookup = + this.banService === undefined + ? shipStorage + : new BanAwareSHIPStorage(shipStorage, this.banService, this.logger) + const slapStorageForLookup = + this.banService === undefined + ? slapStorage + : new BanAwareSLAPStorage(slapStorage, this.banService, this.logger) + + this.services.ls_ship = new ResourceBoundedLookupWrapper( + new DiscoveryServices.SHIPLookupService(shipStorageForLookup as any), + maxLookupResults + ) + this.services.ls_slap = new ResourceBoundedLookupWrapper( + new DiscoveryServices.SLAPLookupService(slapStorageForLookup as any), + maxLookupResults + ) this.logger.log(chalk.blue('Configured lookup service ls_ship with MongoDB')) this.logger.log(chalk.blue('Configured lookup service ls_slap with MongoDB')) } @@ -919,7 +986,8 @@ export default class OverlayExpress { this.engineConfig.suppressDefaultSyncAdvertisements ?? true, this.buildTopicAnchorHeaderResolver(), this.engineConfig.enableBASMSync ?? this.enableBASMSync, - this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks + this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks, + maxLookupResults ) this.initServerWallet() @@ -927,26 +995,36 @@ export default class OverlayExpress { } /** Wrap SHIP/SLAP managers and services with ban-aware filters if BanService is configured. */ - private wrapBanAwareServices (): void { + private wrapBanAwareServices(): void { if (this.banService === undefined) return for (const key of ['tm_ship', 'tm_slap'] as const) { if (this.managers[key] !== undefined) { const label = key === 'tm_ship' ? 'SHIP' : 'SLAP' - this.managers[key] = new BanAwareTopicManager(this.managers[key], this.banService, label, this.logger) + this.managers[key] = new BanAwareTopicManager( + this.managers[key], + this.banService, + label, + this.logger + ) this.logger.log(chalk.blue(`${label} topic manager wrapped with ban-aware filter.`)) } } for (const key of ['ls_ship', 'ls_slap'] as const) { if (this.services[key] !== undefined) { const label = key === 'ls_ship' ? 'SHIP' : 'SLAP' - this.services[key] = new BanAwareLookupWrapper(this.services[key], this.banService, label, this.logger) + this.services[key] = new BanAwareLookupWrapper( + this.services[key], + this.banService, + label, + this.logger + ) this.logger.log(chalk.blue(`${label} lookup service wrapped with ban-aware filter.`)) } } } /** Build the sync config based on enableGASPSync and engineConfig. */ - private buildSyncConfig (): SyncConfigurationMap { + private buildSyncConfig(): SyncConfigurationMap { if (this.enableGASPSync) { return this.engineConfig.syncConfiguration ?? {} } @@ -958,7 +1036,7 @@ export default class OverlayExpress { } /** Build the configured transaction propagation provider chain. */ - private buildBroadcaster (): Broadcaster | undefined { + private buildBroadcaster(): Broadcaster | undefined { const providers: NamedBroadcaster[] = [] const callbackUrl = `https://${this.advertisableFQDN}/arc-ingest` @@ -994,7 +1072,7 @@ export default class OverlayExpress { return new ProviderChainBroadcaster(providers) } - private ensureArcadeProvider (): ArcadeProvider | undefined { + private ensureArcadeProvider(): ArcadeProvider | undefined { if (this.arcadeProvider !== undefined) return this.arcadeProvider if (typeof this.arcadeUrl !== 'string' || this.arcadeUrl.length === 0) return undefined this.arcadeProvider = new ArcadeProvider(this.arcadeUrl, { @@ -1006,7 +1084,7 @@ export default class OverlayExpress { return this.arcadeProvider } - private async fetchArcadeProof (txid: string): Promise { + private async fetchArcadeProof(txid: string): Promise { const provider = this.ensureArcadeProvider() if (provider === undefined) return undefined const proof = await provider.fetchMerkleProof(txid) @@ -1021,7 +1099,9 @@ export default class OverlayExpress { } const valid = await chainTracker.isValidRootForHeight(proof.merkleRoot, blockHeight) if (!valid) { - throw new Error(`Arcade proof for ${txid} did not match the chain tracker at height ${blockHeight}`) + throw new Error( + `Arcade proof for ${txid} did not match the chain tracker at height ${blockHeight}` + ) } return { ...proof, @@ -1029,7 +1109,9 @@ export default class OverlayExpress { } } - private async fetchConfiguredMerkleProof (txid: string): Promise<{ merklePath: MerklePath, blockHeight?: number } | undefined> { + private async fetchConfiguredMerkleProof( + txid: string + ): Promise<{ merklePath: MerklePath; blockHeight?: number } | undefined> { const proof = await this.fetchArcadeProof(txid) if (proof === undefined) return undefined return { @@ -1039,21 +1121,26 @@ export default class OverlayExpress { } /** Build the BASM block header resolver. */ - private buildTopicAnchorHeaderResolver (): TopicAnchorHeaderResolver | undefined { + private buildTopicAnchorHeaderResolver(): TopicAnchorHeaderResolver | undefined { const configured = this.engineConfig.topicAnchorHeaderResolver ?? this.topicAnchorHeaderResolver if (configured !== undefined) { return configured } return async (blockHeight: number) => { - const response = await fetch(`https://api.whatsonchain.com/v1/bsv/${this.network}/block/${blockHeight}/header`, { - method: 'GET', - headers: { Accept: 'application/json' } - }) + const response = await fetch( + `https://api.whatsonchain.com/v1/bsv/${this.network}/block/${blockHeight}/header`, + { + method: 'GET', + headers: { Accept: 'application/json' } + } + ) if (!response.ok) { - throw new Error(`WhatsOnChain header lookup failed for height ${blockHeight}: ${response.status}`) + throw new Error( + `WhatsOnChain header lookup failed for height ${blockHeight}: ${response.status}` + ) } - const header = await response.json() as { hash?: string, merkleroot?: string } + const header = (await response.json()) as { hash?: string; merkleroot?: string } if (typeof header.hash !== 'string') { throw new TypeError(`WhatsOnChain did not return a block hash for height ${blockHeight}`) } @@ -1066,17 +1153,18 @@ export default class OverlayExpress { } /** Resolve the SLAP trackers from config or network defaults. */ - private resolveSlapTrackers (): string[] | undefined { + private resolveSlapTrackers(): string[] | undefined { if (Array.isArray(this.engineConfig.slapTrackers)) return this.engineConfig.slapTrackers return this.network === 'test' ? DEFAULT_TESTNET_SLAP_TRACKERS : DEFAULT_SLAP_TRACKERS } /** Build the WalletAdvertiser (or use user-provided one). */ - private async buildAdvertiser (): Promise { + private async buildAdvertiser(): Promise { if (this.engineConfig.advertiser !== undefined) return this.engineConfig.advertiser - const storageBase = this.network === 'test' - ? 'https://staging-storage.babbage.systems' - : 'https://storage.babbage.systems' + const storageBase = + this.network === 'test' + ? 'https://staging-storage.babbage.systems' + : 'https://storage.babbage.systems' try { return new DiscoveryServices.WalletAdvertiser( this.network, @@ -1085,13 +1173,15 @@ export default class OverlayExpress { `https://${this.advertisableFQDN}` ) } catch (e) { - this.logger.log(`Advertiser not initialized for FQDN ${this.advertisableFQDN} - SHIP and SLAP will be disabled. Reason: ${e}`) + this.logger.log( + `Advertiser not initialized for FQDN ${this.advertisableFQDN} - SHIP and SLAP will be disabled. Reason: ${e}` + ) return undefined } } /** Initialize the server wallet for BSV mutual authentication. */ - private initServerWallet (): void { + private initServerWallet(): void { try { const keyDeriver = new KeyDeriver(new PrivateKey(this.privateKey, 'hex')) const storageManager = new WalletStorageManager(keyDeriver.identityKey) @@ -1101,7 +1191,11 @@ export default class OverlayExpress { this.adminIdentityKey ??= keyDeriver.identityKey this.logger.log(chalk.blue('Server wallet initialized for BSV mutual authentication.')) } catch (e) { - this.logger.log(chalk.yellow(`Server wallet could not be initialized. BSV auth will not be available. Reason: ${e}`)) + this.logger.log( + chalk.yellow( + `Server wallet could not be initialized. BSV auth will not be available. Reason: ${e}` + ) + ) } } @@ -1109,9 +1203,11 @@ export default class OverlayExpress { * Ensures that Knex is configured and returns it. * @throws Error if Knex is not configured */ - private ensureKnex (): Knex.Knex { + private ensureKnex(): Knex.Knex { if (this.knex === undefined) { - throw new TypeError('You must configure your SQL database with the .configureKnex() method first!') + throw new TypeError( + 'You must configure your SQL database with the .configureKnex() method first!' + ) } return this.knex } @@ -1120,9 +1216,11 @@ export default class OverlayExpress { * Ensures that MongoDB is configured and returns it. * @throws Error if MongoDB is not configured */ - private ensureMongo (): Db { + private ensureMongo(): Db { if (this.mongoDb === undefined) { - throw new TypeError('You must configure your MongoDB connection with the .configureMongo() method first!') + throw new TypeError( + 'You must configure your MongoDB connection with the .configureMongo() method first!' + ) } return this.mongoDb } @@ -1131,9 +1229,11 @@ export default class OverlayExpress { * Ensures that the Overlay Engine is configured and returns it. * @throws Error if the Engine is not configured */ - private ensureEngine (): Engine { + private ensureEngine(): Engine { if (this.engine === undefined) { - throw new TypeError('You must configure your Overlay Services engine with the .configureEngine() method first!') + throw new TypeError( + 'You must configure your Overlay Services engine with the .configureEngine() method first!' + ) } return this.engine } @@ -1141,8 +1241,10 @@ export default class OverlayExpress { /** * Creates a JanitorService instance with current configuration. */ - private createJanitor (): JanitorService { + private createJanitor(): JanitorService { const mongoDb = this.ensureMongo() + const prefix = this.edgePolicyConfig.environmentPrefix + const profile = readResourceProfile(prefix) return new JanitorService({ mongoDb, logger: this.logger, @@ -1150,12 +1252,32 @@ export default class OverlayExpress { hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore, banService: this.banService, autoBanOnRemoval: this.janitorConfig.autoBanOnRemoval, - allowPrivateHosts: this.janitorConfig.allowPrivateHosts + allowPrivateHosts: this.janitorConfig.allowPrivateHosts, + batchSize: + this.janitorConfig.batchSize ?? + readResourceLimit( + prefix, + 'JANITOR_BATCH_SIZE', + profileValue(profile, { small: 100, standard: 250, highThroughput: 1000 }), + 100_000 + ), + maxReportResults: + this.janitorConfig.maxReportResults ?? + readResourceLimit( + prefix, + 'JANITOR_MAX_REPORT_RESULTS', + profileValue(profile, { small: 500, standard: 1000, highThroughput: 5000 }), + 1_000_000 + ) }) } /** Ban a domain and remove all its SHIP/SLAP records from MongoDB. */ - private async handleBanDomain (res: express.Response, value: string, reason?: string): Promise { + private async handleBanDomain( + res: express.Response, + value: string, + reason?: string + ): Promise { await this.banService!.banDomain(value, reason) const db = this.ensureMongo() const [shipDeleted, slapDeleted] = await Promise.all([ @@ -1169,10 +1291,17 @@ export default class OverlayExpress { } /** Parse outpoint string, ban it, and evict it from all lookup services. */ - private async handleBanOutpoint (res: express.Response, engine: Engine, value: string, reason?: string): Promise { + private async handleBanOutpoint( + res: express.Response, + engine: Engine, + value: string, + reason?: string + ): Promise { const dotIndex = value.lastIndexOf('.') if (dotIndex === -1) { - return res.status(400).json({ status: 'error', message: 'Outpoint format must be "txid.outputIndex"' }) + return res + .status(400) + .json({ status: 'error', message: 'Outpoint format must be "txid.outputIndex"' }) } const txid = value.substring(0, dotIndex) const outputIndex = Number.parseInt(value.substring(dotIndex + 1)) @@ -1188,19 +1317,31 @@ export default class OverlayExpress { } /** Evict an output from a specific service or all services (silent per-service errors). */ - private async evictFromServices (engine: Engine, txid: string, outputIndex: number, service?: string): Promise { + private async evictFromServices( + engine: Engine, + txid: string, + outputIndex: number, + service?: string + ): Promise { if (typeof service === 'string') { const svc = engine.lookupServices[service] if (svc !== undefined) await svc.outputEvicted(txid, outputIndex) return } for (const svc of Object.values(engine.lookupServices)) { - try { await svc.outputEvicted(txid, outputIndex) } catch { /* best-effort */ } + try { + await svc.outputEvicted(txid, outputIndex) + } catch { + /* best-effort */ + } } } /** Look up the domain of an outpoint from SHIP or SLAP records. */ - private async lookupDomainForOutpoint (txid: string, outputIndex: number): Promise { + private async lookupDomainForOutpoint( + txid: string, + outputIndex: number + ): Promise { const db = this.ensureMongo() const [shipRecord, slapRecord] = await Promise.all([ db.collection('shipRecords').findOne({ txid, outputIndex }), @@ -1210,7 +1351,7 @@ export default class OverlayExpress { } /** Ban a domain and delete all SHIP/SLAP records for it. */ - private async banDomainAndRemoveRecords (domain: string, reason: string): Promise { + private async banDomainAndRemoveRecords(domain: string, reason: string): Promise { await this.banService!.banDomain(domain, reason) const db = this.ensureMongo() await Promise.all([ @@ -1219,8 +1360,10 @@ export default class OverlayExpress { ]) } - private async runHealthCheck ( - definition: Required> & { handler: HealthCheckHandler } + private async runHealthCheck( + definition: Required> & { + handler: HealthCheckHandler + } ): Promise { const startedAt = Date.now() let timeout: ReturnType | undefined @@ -1268,8 +1411,12 @@ export default class OverlayExpress { } } - private async collectHealthReport (mode: 'live' | 'ready' | 'full'): Promise { - const definitions: Array> & { handler: HealthCheckHandler }> = [ + private async collectHealthReport(mode: 'live' | 'ready' | 'full'): Promise { + const definitions: Array< + Required> & { + handler: HealthCheckHandler + } + > = [ { name: 'process', scope: 'live', @@ -1346,7 +1493,7 @@ export default class OverlayExpress { }) } - const filteredDefinitions = definitions.filter((definition) => { + const filteredDefinitions = definitions.filter(definition => { if (mode === 'full') { return true } @@ -1354,7 +1501,9 @@ export default class OverlayExpress { return definition.scope === mode }) - const checks = await Promise.all(filteredDefinitions.map(async definition => await this.runHealthCheck(definition))) + const checks = await Promise.all( + filteredDefinitions.map(async definition => await this.runHealthCheck(definition)) + ) const liveChecks = checks.filter(check => check.scope === 'live') const readyChecks = checks.filter(check => check.scope === 'ready') const live = liveChecks.every(check => !check.critical || check.status === 'ok') @@ -1367,9 +1516,10 @@ export default class OverlayExpress { status = 'degraded' } - const context = typeof this.healthConfig.contextProvider === 'function' - ? await this.healthConfig.contextProvider() - : undefined + const context = + typeof this.healthConfig.contextProvider === 'function' + ? await this.healthConfig.contextProvider() + : undefined const report: HealthReport = { status, @@ -1401,21 +1551,25 @@ export default class OverlayExpress { /** * Renders a request or response body for verbose logging, truncating overly long payloads. */ - private formatBodyForLog (body: any, okPrefix: string): string { + private formatBodyForLog(body: any, okPrefix: string): string { if (Buffer.isBuffer(body)) { return chalk.green(`${okPrefix} binary body (${serializeLogValue(body.byteLength)} bytes)`) } if (typeof body === 'string') { - return chalk.green(`${okPrefix} string body (${serializeLogValue(Buffer.byteLength(body, 'utf8'))} bytes)`) + return chalk.green( + `${okPrefix} string body (${serializeLogValue(Buffer.byteLength(body, 'utf8'))} bytes)` + ) } if (body != null && typeof body === 'object') { const keys = Array.isArray(body) ? body.length : Object.keys(body).length - return chalk.green(`${okPrefix} structured body (${serializeLogValue(keys)} top-level item(s))`) + return chalk.green( + `${okPrefix} structured body (${serializeLogValue(keys)} top-level item(s))` + ) } return chalk.green(`${okPrefix} type=${serializeLogValue(typeof body)}`) } - private redactHeadersForLog (headers: Record): Record { + private redactHeadersForLog(headers: Record): Record { const sensitiveHeader = /authorization|cookie|token|secret|payment|signature|nonce/i return Object.fromEntries( Object.entries(headers).map(([name, value]) => [ @@ -1499,13 +1653,19 @@ export default class OverlayExpress { /** * Installs middleware that verbosely logs incoming requests and outgoing responses. */ - private setupVerboseRequestLogging (): void { + private setupVerboseRequestLogging(): void { this.app.use((req, res, next) => { const startTime = Date.now() // Log incoming request details - this.logger.log(chalk.magenta.bold(`Incoming Request: method=${serializeLogValue(req.method)} url=${serializeLogValue(req.originalUrl)}`)) - this.logger.log(chalk.cyan(`Headers: ${serializeLogValue(this.redactHeadersForLog(req.headers))}`)) + this.logger.log( + chalk.magenta.bold( + `Incoming Request: method=${serializeLogValue(req.method)} url=${serializeLogValue(req.originalUrl)}` + ) + ) + this.logger.log( + chalk.cyan(`Headers: ${serializeLogValue(this.redactHeadersForLog(req.headers))}`) + ) // Handle request body if (req.body != null && Object.keys(req.body).length > 0) { @@ -1529,7 +1689,11 @@ export default class OverlayExpress { `Outgoing Response: method=${serializeLogValue(req.method)} url=${serializeLogValue(req.originalUrl)} status=${serializeLogValue(res.statusCode)} durationMs=${serializeLogValue(duration)}` ) ) - this.logger.log(chalk.cyan(`Response Headers: ${serializeLogValue(this.redactHeadersForLog(res.getHeaders()))}`)) + this.logger.log( + chalk.cyan( + `Response Headers: ${serializeLogValue(this.redactHeadersForLog(res.getHeaders()))}` + ) + ) // Handle response body if (responseBody != null) { @@ -1545,41 +1709,138 @@ export default class OverlayExpress { * Starts the Express server. * Sets up routes and begins listening on the configured port. */ - async start (): Promise { + async start(): Promise { const engine = this.ensureEngine() const knex = this.ensureKnex() this.startTime = new Date() const edgePolicy = this.edgePolicyConfig - this.app.disable('x-powered-by') - this.app.use(securityHeaders({ - environmentPrefix: edgePolicy.environmentPrefix, - ...edgePolicy.securityHeaders - })) - this.app.use(corsPolicy({ - environmentPrefix: edgePolicy.environmentPrefix, - allowedOrigins: edgePolicy.allowedOrigins, - methods: ['GET', 'POST', 'OPTIONS'] - })) - this.app.use(concurrencyLimit( + const resourceProfile = readResourceProfile(edgePolicy.environmentPrefix) + const maxResponseBytes = readResourceLimit( + edgePolicy.environmentPrefix, + 'MAX_RESPONSE_BYTES', + profileValue(resourceProfile, { + small: 4 * 1024 * 1024, + standard: 8 * 1024 * 1024, + highThroughput: 32 * 1024 * 1024 + }), + 512 * 1024 * 1024 + ) + const maxBasmTxids = readResourceLimit( + edgePolicy.environmentPrefix, + 'MAX_BASM_TXIDS', + profileValue(resourceProfile, { small: 500, standard: 1000, highThroughput: 5000 }), + 1_000_000 + ) + const maxBasmAnchorRange = readResourceLimit( + edgePolicy.environmentPrefix, + 'MAX_BASM_ANCHOR_RANGE', + profileValue(resourceProfile, { small: 500, standard: 1000, highThroughput: 5000 }), + 1_000_000 + ) + const adminListDefaultLimit = readResourceLimit( edgePolicy.environmentPrefix, - edgePolicy.maxConcurrentRequests - )) - this.app.use(bodyParser.json({ - limit: readBodyLimitBytes( - `${edgePolicy.environmentPrefix}_JSON`, - edgePolicy.jsonBodyLimitBytes - ), - type: 'application/json' - })) - this.app.use(bodyParser.raw({ - limit: readBodyLimitBytes( - `${edgePolicy.environmentPrefix}_BINARY`, - edgePolicy.binaryBodyLimitBytes - ), - type: 'application/octet-stream' - })) + 'ADMIN_LIST_DEFAULT_LIMIT', + profileValue(resourceProfile, { small: 25, standard: 50, highThroughput: 100 }), + 1_000_000 + ) + const adminListMaxLimit = readResourceLimit( + edgePolicy.environmentPrefix, + 'ADMIN_LIST_MAX_LIMIT', + profileValue(resourceProfile, { small: 100, standard: 200, highThroughput: 1000 }), + 1_000_000 + ) + const adminListMaxOffset = readResourceLimit( + edgePolicy.environmentPrefix, + 'ADMIN_LIST_MAX_OFFSET', + profileValue(resourceProfile, { + small: 10_000, + standard: 100_000, + highThroughput: 1_000_000 + }), + 100_000_000 + ) + if ( + adminListDefaultLimit !== -1 && + adminListMaxLimit !== -1 && + adminListDefaultLimit > adminListMaxLimit + ) { + throw new TypeError( + 'OVERLAY_ADMIN_LIST_DEFAULT_LIMIT cannot exceed OVERLAY_ADMIN_LIST_MAX_LIMIT' + ) + } + const parseAdminPage = ( + rawPageValue: unknown, + rawLimitValue: unknown + ): { page: number; limit: number; skip: number } => { + const rawPage = Number.parseInt(typeof rawPageValue === 'string' ? rawPageValue : '', 10) + const requestedPage = Math.max(1, Number.isNaN(rawPage) ? 1 : rawPage) + const rawLimitText = + typeof rawLimitValue === 'string' ? rawLimitValue.trim().toLowerCase() : '' + const parsedLimit = + rawLimitText === '-1' || rawLimitText === 'unlimited' + ? -1 + : Number.parseInt(rawLimitText, 10) + const requestedLimit = + rawLimitText.length === 0 || Number.isNaN(parsedLimit) ? adminListDefaultLimit : parsedLimit + if (requestedLimit !== -1 && (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1)) { + throw new TypeError('limit must be a positive integer, -1, or unlimited') + } + let limit = requestedLimit + if (adminListMaxLimit !== -1) { + limit = + requestedLimit === -1 ? adminListMaxLimit : Math.min(requestedLimit, adminListMaxLimit) + } + const page = limit === -1 ? 1 : requestedPage + const skip = limit === -1 ? 0 : (page - 1) * limit + if (!Number.isSafeInteger(skip) || (adminListMaxOffset !== -1 && skip > adminListMaxOffset)) { + throw new TypeError('requested page exceeds the configured maximum offset') + } + return { page, limit, skip } + } + this.app.disable('x-powered-by') + this.app.use(initialDoubleSlashCompatibility) + this.app.use( + securityHeaders({ + environmentPrefix: edgePolicy.environmentPrefix, + ...edgePolicy.securityHeaders + }) + ) + this.app.use( + corsPolicy({ + environmentPrefix: edgePolicy.environmentPrefix, + allowedOrigins: edgePolicy.allowedOrigins, + methods: ['GET', 'POST', 'OPTIONS'] + }) + ) + this.app.use( + concurrencyLimit( + edgePolicy.environmentPrefix, + edgePolicy.maxConcurrentRequests === 200 + ? profileValue(resourceProfile, { small: 8, standard: 24, highThroughput: 96 }) + : edgePolicy.maxConcurrentRequests + ) + ) + this.app.use( + bodyParser.json({ + limit: readBodyLimitBytes( + `${edgePolicy.environmentPrefix}_JSON`, + edgePolicy.jsonBodyLimitBytes + ), + type: 'application/json' + }) + ) + this.app.use( + bodyParser.raw({ + limit: readBodyLimitBytes( + `${edgePolicy.environmentPrefix}_BINARY`, + edgePolicy.binaryBodyLimitBytes + ), + type: 'application/octet-stream' + }) + ) this.app.use(bodyParserErrorHandler) + this.app.use(responseSizeLimit(edgePolicy.environmentPrefix, maxResponseBytes)) if (this.verboseRequestLogging) { this.setupVerboseRequestLogging() @@ -1588,18 +1849,20 @@ export default class OverlayExpress { // Serve a static documentation site or user interface this.app.get('/', (req, res) => { res.set('content-type', 'text/html') - res.send(makeUserInterface({ - ...this.webUIConfig, - adminIdentityKey: this.adminIdentityKey - })) + res.send( + makeUserInterface({ + ...this.webUIConfig, + adminIdentityKey: this.adminIdentityKey + }) + ) }) // Serve health check endpoints this.app.get('/health/live', (_, res) => { - ; (async () => { + ;(async () => { const report = await this.collectHealthReport('live') return res.status(report.live ? 200 : 503).json(report) - })().catch((error) => { + })().catch(error => { this.logger.error({ operation: 'overlay.health_live', error }) res.status(500).json({ status: 'error', @@ -1608,11 +1871,22 @@ export default class OverlayExpress { }) }) + // Compatibility alias used by Kubernetes probes and existing deployments. + this.app.get('/healthz', (_, res) => { + ;(async () => { + const report = await this.collectHealthReport('live') + return res.status(report.live ? 200 : 503).json(report) + })().catch(error => { + this.logger.error({ operation: 'overlay.healthz', error }) + res.status(500).json({ status: 'error', message: 'Health report unavailable' }) + }) + }) + this.app.get('/health/ready', (_, res) => { - ; (async () => { + ;(async () => { const report = await this.collectHealthReport('ready') return res.status(report.ready ? 200 : 503).json(report) - })().catch((error) => { + })().catch(error => { this.logger.error({ operation: 'overlay.health_ready', error }) res.status(500).json({ status: 'error', @@ -1622,10 +1896,10 @@ export default class OverlayExpress { }) this.app.get('/health', (_, res) => { - ; (async () => { + ;(async () => { const report = await this.collectHealthReport('full') return res.status(report.ready ? 200 : 503).json(report) - })().catch((error) => { + })().catch(error => { this.logger.error({ operation: 'overlay.health_full', error }) res.status(500).json({ status: 'error', @@ -1636,7 +1910,7 @@ export default class OverlayExpress { // List hosted topic managers and lookup services this.app.get('/listTopicManagers', (_, res) => { - ; (async () => { + ;(async () => { try { const result = await engine.listTopicManagers() return res.status(200).json(result) @@ -1655,7 +1929,7 @@ export default class OverlayExpress { }) this.app.get('/listLookupServiceProviders', (_, res) => { - ; (async () => { + ;(async () => { try { const result = await engine.listLookupServiceProviders() return res.status(200).json(result) @@ -1675,7 +1949,7 @@ export default class OverlayExpress { // Host documentation for the services this.app.get('/getDocumentationForTopicManager', (req, res) => { - ; (async () => { + ;(async () => { try { const manager = req.query.manager as string const result = await engine.getDocumentationForTopicManager(manager) @@ -1696,7 +1970,7 @@ export default class OverlayExpress { }) this.app.get('/getDocumentationForLookupServiceProvider', (req, res) => { - ; (async () => { + ;(async () => { try { const lookupService = req.query.lookupService as string const result = await engine.getDocumentationForLookupServiceProvider(lookupService) @@ -1718,7 +1992,7 @@ export default class OverlayExpress { // Submit transactions and facilitate lookup requests this.app.post('/submit', (req, res) => { - ; (async () => { + ;(async () => { try { // Parse out the topics and construct the tagged BEEF const topicsHeader = req.headers['x-topics'] @@ -1747,10 +2021,15 @@ export default class OverlayExpress { // Using a callback function, we can return once the STEAK is ready let responseSent = false - const steak = await engine.submit(taggedBEEF, (steak: STEAK) => { - responseSent = true - return res.status(200).json(steak) - }, 'current-tx', offChainValues) + const steak = await engine.submit( + taggedBEEF, + (steak: STEAK) => { + responseSent = true + return res.status(200).json(steak) + }, + 'current-tx', + offChainValues + ) if (!responseSent) { res.status(200).json(steak) } @@ -1770,14 +2049,14 @@ export default class OverlayExpress { }) this.app.post('/lookup', (req, res) => { - ; (async () => { + ;(async () => { try { // Check for aggregation header to determine response format const aggregationHeader = req.headers['x-aggregation'] const shouldReturnBinary = aggregationHeader === 'yes' // Validate request body structure - const lookupRequest = req.body as { service: string, query: unknown } + const lookupRequest = req.body as { service: string; query: unknown } if (typeof lookupRequest.service !== 'string' || lookupRequest.query === undefined) { return res.status(400).json({ status: 'error', @@ -1809,7 +2088,7 @@ export default class OverlayExpress { // Write outputIndex writer.writeVarIntNum(output.outputIndex) // Write context length and data - if ((output.context != null) && output.context.length > 0) { + if (output.context != null && output.context.length > 0) { writer.writeVarIntNum(output.context.length) writer.write(output.context) } else { @@ -1844,11 +2123,13 @@ export default class OverlayExpress { (typeof this.arcadeUrl === 'string' && this.arcadeUrl.length > 0) ) { this.app.post('/arc-ingest', (req, res) => { - ; (async () => { + ;(async () => { try { return await this.processArcIngest(engine, req, res) } catch (error) { - this.logger.error(chalk.red(`Error in /arc-ingest: error=${serializeErrorForLog(error)}`)) + this.logger.error( + chalk.red(`Error in /arc-ingest: error=${serializeErrorForLog(error)}`) + ) return res.status(400).json({ status: 'error', message: publicErrorMessage(error) @@ -1862,13 +2143,15 @@ export default class OverlayExpress { }) }) } else { - this.logger.warn(chalk.yellow('Disabling ARC/Arcade ingest because no provider was configured.')) + this.logger.warn( + chalk.yellow('Disabling ARC/Arcade ingest because no provider was configured.') + ) } // GASP sync routes if enabled if (this.enableGASPSync) { this.app.post('/requestSyncResponse', (req, res) => { - ; (async () => { + ;(async () => { try { const topic = req.headers['x-bsv-topic'] as string const response = await engine.provideForeignSyncResponse(req.body, topic) @@ -1889,7 +2172,7 @@ export default class OverlayExpress { }) this.app.post('/requestForeignGASPNode', (req, res) => { - ; (async () => { + ;(async () => { try { const { graphID, txid, outputIndex } = req.body const response = await engine.provideForeignGASPNode(graphID, txid, outputIndex) @@ -1934,36 +2217,55 @@ export default class OverlayExpress { handler: (req: express.Request) => Promise, ...middleware: express.RequestHandler[] ): void => { - this.app.post(path, ...(middleware as any[]), (req: express.Request, res: express.Response) => { - ; (async () => { - try { - return res.status(200).json(await handler(req)) - } catch (error) { - console.error(chalk.red(`Error in ${path}:`), error) - return res.status(400).json({ - status: 'error', - message: publicErrorMessage(error) - }) - } - })().catch(() => { - res.status(500).json({ status: 'error', message: 'Unexpected error' }) - }) - }) + this.app.post( + path, + ...(middleware as any[]), + (req: express.Request, res: express.Response) => { + ;(async () => { + try { + return res.status(200).json(await handler(req)) + } catch (error) { + console.error(chalk.red(`Error in ${path}:`), error) + return res.status(400).json({ + status: 'error', + message: publicErrorMessage(error) + }) + } + })().catch(() => { + res.status(500).json({ status: 'error', message: 'Unexpected error' }) + }) + } + ) } const requireTxids = (value: unknown): string[] => { if (!Array.isArray(value) || !value.every(txid => typeof txid === 'string')) { throw new PublicRequestError('txids must be an array of strings') } + if (maxBasmTxids !== -1 && value.length > maxBasmTxids) { + throw new PublicRequestError(`txids must contain at most ${maxBasmTxids} entries`) + } return value } - registerJsonRoute('/requestTopicAnchorTip', async req => - await basmEngine.provideTopicAnchorTip(readBasmTopic(req))) + registerJsonRoute( + '/requestTopicAnchorTip', + async req => await basmEngine.provideTopicAnchorTip(readBasmTopic(req)) + ) registerJsonRoute('/requestTopicAnchorRange', async req => { const { fromHeight, toHeight } = req.body - return await basmEngine.provideTopicAnchorRange(readBasmTopic(req), Number(fromHeight), Number(toHeight)) + const from = Number(fromHeight) + const to = Number(toHeight) + if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to) || from < 0 || to < from) { + throw new PublicRequestError('fromHeight and toHeight must define a valid ascending range') + } + if (maxBasmAnchorRange !== -1 && to - from + 1 > maxBasmAnchorRange) { + throw new PublicRequestError( + `topic anchor range must contain at most ${maxBasmAnchorRange} blocks` + ) + } + return await basmEngine.provideTopicAnchorRange(readBasmTopic(req), from, to) }) registerJsonRoute('/requestAdmittedList', async req => { @@ -1978,11 +2280,17 @@ export default class OverlayExpress { registerJsonRoute('/requestCompoundMerklePath', async req => { const topic = readBasmTopic(req) const { blockHeight, txids } = req.body - return await basmEngine.provideCompoundMerklePath(topic, Number(blockHeight), requireTxids(txids)) + return await basmEngine.provideCompoundMerklePath( + topic, + Number(blockHeight), + requireTxids(txids) + ) }) - registerJsonRoute('/requestRawTransactions', async req => - await basmEngine.provideRawTransactions(requireTxids(req.body.txids))) + registerJsonRoute( + '/requestRawTransactions', + async req => await basmEngine.provideRawTransactions(requireTxids(req.body.txids)) + ) /** * ============== ADMIN ROUTES ============== @@ -2012,7 +2320,11 @@ export default class OverlayExpress { * 1. Bearer token (Authorization: Bearer ) - for cron jobs, scripts, and fallback * 2. BSV mutual auth - if req.auth.identityKey matches the admin identity key */ - const checkAdminAuth = (req: express.Request, res: express.Response, next: express.NextFunction): void => { + const checkAdminAuth = ( + req: express.Request, + res: express.Response, + next: express.NextFunction + ): void => { // Method 1: BSV mutual authentication (identity key match) const authReq = req as unknown as AuthRequest if ( @@ -2038,7 +2350,10 @@ export default class OverlayExpress { return } - res.status(401).json({ status: 'error', message: 'Unauthorized: Provide a Bearer token or authenticate with your wallet' }) + res.status(401).json({ + status: 'error', + message: 'Unauthorized: Provide a Bearer token or authenticate with your wallet' + }) } /** @@ -2058,7 +2373,7 @@ export default class OverlayExpress { * Admin route: Get server statistics and overview. */ this.app.get('/admin/stats', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { const db = this.ensureMongo() @@ -2102,17 +2417,13 @@ export default class OverlayExpress { * Admin route: List all SHIP records with full details. */ this.app.get('/admin/ship-records', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { const db = this.ensureMongo() const collection = db.collection('shipRecords') const search = typeof req.query.search === 'string' ? req.query.search : undefined - const rawPage = Number.parseInt(req.query.page as string, 10) - const page = Math.max(1, Number.isNaN(rawPage) ? 1 : rawPage) - const rawLimit = Number.parseInt(req.query.limit as string, 10) - const limit = Math.min(200, Math.max(1, Number.isNaN(rawLimit) ? 50 : rawLimit)) - const skip = (page - 1) * limit + const { page, limit, skip } = parseAdminPage(req.query.page, req.query.limit) const query: any = {} if (typeof search === 'string' && search.length > 0) { @@ -2125,13 +2436,23 @@ export default class OverlayExpress { } const [records, total] = await Promise.all([ - collection.find(query).sort({ createdAt: -1 }).skip(skip).limit(limit).toArray(), + (() => { + let cursor = collection.find(query).sort({ createdAt: -1 }).skip(skip) + if (limit !== -1) cursor = cursor.limit(limit) + return cursor.toArray() + })(), collection.countDocuments(query) ]) return res.status(200).json({ status: 'success', - data: { records, total, page, limit, pages: Math.ceil(total / limit) } + data: { + records, + total, + page, + limit, + pages: limit === -1 ? 1 : Math.ceil(total / limit) + } }) } catch (error) { return res.status(400).json({ @@ -2148,17 +2469,13 @@ export default class OverlayExpress { * Admin route: List all SLAP records with full details. */ this.app.get('/admin/slap-records', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { const db = this.ensureMongo() const collection = db.collection('slapRecords') const search = typeof req.query.search === 'string' ? req.query.search : undefined - const rawPage = Number.parseInt(req.query.page as string, 10) - const page = Math.max(1, Number.isNaN(rawPage) ? 1 : rawPage) - const rawLimit = Number.parseInt(req.query.limit as string, 10) - const limit = Math.min(200, Math.max(1, Number.isNaN(rawLimit) ? 50 : rawLimit)) - const skip = (page - 1) * limit + const { page, limit, skip } = parseAdminPage(req.query.page, req.query.limit) const query: any = {} if (typeof search === 'string' && search.length > 0) { @@ -2171,13 +2488,23 @@ export default class OverlayExpress { } const [records, total] = await Promise.all([ - collection.find(query).sort({ createdAt: -1 }).skip(skip).limit(limit).toArray(), + (() => { + let cursor = collection.find(query).sort({ createdAt: -1 }).skip(skip) + if (limit !== -1) cursor = cursor.limit(limit) + return cursor.toArray() + })(), collection.countDocuments(query) ]) return res.status(200).json({ status: 'success', - data: { records, total, page, limit, pages: Math.ceil(total / limit) } + data: { + records, + total, + page, + limit, + pages: limit === -1 ? 1 : Math.ceil(total / limit) + } }) } catch (error) { return res.status(400).json({ @@ -2194,7 +2521,7 @@ export default class OverlayExpress { * Admin route: Check health of a specific URL. */ this.app.post('/admin/health-check', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { const url = req.body?.url if (typeof url !== 'string' || url.length === 0) { @@ -2218,15 +2545,20 @@ export default class OverlayExpress { * Admin route: Ban a domain or outpoint. */ this.app.post('/admin/ban', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { if (this.banService === undefined) { - return res.status(400).json({ status: 'error', message: 'Ban service not available (MongoDB not configured)' }) + return res.status(400).json({ + status: 'error', + message: 'Ban service not available (MongoDB not configured)' + }) } const { type, value, reason } = req.body if (type !== 'domain' && type !== 'outpoint') { - return res.status(400).json({ status: 'error', message: 'type must be "domain" or "outpoint"' }) + return res + .status(400) + .json({ status: 'error', message: 'type must be "domain" or "outpoint"' }) } if (typeof value !== 'string' || value.length === 0) { return res.status(400).json({ status: 'error', message: 'value is required' }) @@ -2251,21 +2583,25 @@ export default class OverlayExpress { * Admin route: Remove a ban. */ this.app.post('/admin/unban', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { if (this.banService === undefined) { return res.status(400).json({ status: 'error', message: 'Ban service not available' }) } - const { type, value } = req.body as { type: unknown, value: unknown } + const { type, value } = req.body as { type: unknown; value: unknown } if (type !== 'domain' && type !== 'outpoint') { - return res.status(400).json({ status: 'error', message: 'type must be "domain" or "outpoint"' }) + return res + .status(400) + .json({ status: 'error', message: 'type must be "domain" or "outpoint"' }) } if (typeof value !== 'string' || value.length === 0) { return res.status(400).json({ status: 'error', message: 'value is required' }) } await this.banService.removeBan(type, value) - return res.status(200).json({ status: 'success', message: `${type} "${String(value)}" unbanned.` }) + return res + .status(200) + .json({ status: 'success', message: `${type} "${String(value)}" unbanned.` }) } catch (error) { return res.status(400).json({ status: 'error', @@ -2281,15 +2617,16 @@ export default class OverlayExpress { * Admin route: List all bans. */ this.app.get('/admin/bans', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { if (this.banService === undefined) { return res.status(200).json({ status: 'success', data: { bans: [] } }) } const type = req.query.type as 'domain' | 'outpoint' | undefined const validType = type === 'domain' || type === 'outpoint' ? type : undefined - const bans = await this.banService.listBans(validType) - return res.status(200).json({ status: 'success', data: { bans } }) + const { page, limit, skip } = parseAdminPage(req.query.page, req.query.limit) + const bans = await this.banService.listBans(validType, limit, skip) + return res.status(200).json({ status: 'success', data: { bans, page, limit } }) } catch (error) { return res.status(400).json({ status: 'error', @@ -2305,11 +2642,14 @@ export default class OverlayExpress { * Admin route: Remove a token by outpoint, optionally banning the domain. */ this.app.post('/admin/remove-token', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { const { txid, outputIndex, service, ban, banDomain: shouldBanDomain } = req.body if (typeof txid !== 'string' || typeof outputIndex !== 'number') { - return res.status(400).json({ status: 'error', message: 'txid (string) and outputIndex (number) are required' }) + return res.status(400).json({ + status: 'error', + message: 'txid (string) and outputIndex (number) are required' + }) } // Look up domain before eviction if needed for banning @@ -2321,17 +2661,30 @@ export default class OverlayExpress { await this.evictFromServices(engine, txid, outputIndex, service) if (ban === true && this.banService !== undefined) { - await this.banService.banOutpoint(txid, outputIndex, 'Manually removed by admin', removedDomain) + await this.banService.banOutpoint( + txid, + outputIndex, + 'Manually removed by admin', + removedDomain + ) } - if (shouldBanDomain === true && typeof removedDomain === 'string' && this.banService !== undefined) { - await this.banDomainAndRemoveRecords(removedDomain, 'Domain banned by admin via token removal') + if ( + shouldBanDomain === true && + typeof removedDomain === 'string' && + this.banService !== undefined + ) { + await this.banDomainAndRemoveRecords( + removedDomain, + 'Domain banned by admin via token removal' + ) } const banMsg = ban === true ? ' Outpoint banned.' : '' - const domainMsg = shouldBanDomain === true && typeof removedDomain === 'string' - ? ` Domain "${removedDomain}" banned.` - : '' + const domainMsg = + shouldBanDomain === true && typeof removedDomain === 'string' + ? ` Domain "${removedDomain}" banned.` + : '' return res.status(200).json({ status: 'success', message: `Token ${txid}.${outputIndex} removed.${banMsg}${domainMsg}` @@ -2351,10 +2704,12 @@ export default class OverlayExpress { * Admin route to manually sync advertisements, calling `engine.syncAdvertisements()`. */ this.app.post('/admin/syncAdvertisements', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { await engine.syncAdvertisements() - return res.status(200).json({ status: 'success', message: 'Advertisements synced successfully' }) + return res + .status(200) + .json({ status: 'success', message: 'Advertisements synced successfully' }) } catch (error) { console.error(chalk.red('Error in /admin/syncAdvertisements:'), error) return res.status(400).json({ @@ -2374,10 +2729,12 @@ export default class OverlayExpress { * Admin route to manually start GASP sync, calling `engine.startGASPSync()`. */ this.app.post('/admin/startGASPSync', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { await engine.startGASPSync() - return res.status(200).json({ status: 'success', message: 'GASP sync started and completed' }) + return res + .status(200) + .json({ status: 'success', message: 'GASP sync started and completed' }) } catch (error) { console.error(chalk.red('Error in /admin/startGASPSync:'), error) return res.status(400).json({ @@ -2396,58 +2753,74 @@ export default class OverlayExpress { /** * Admin route to manually start BASM sync, calling `engine.startBASMSync()`. */ - registerJsonRoute('/admin/startBASMSync', async () => { - const report = await basmEngine.startBASMSync() - return { status: 'success', message: 'BASM sync started and completed', data: report } - }, checkAdminAuth as any) + registerJsonRoute( + '/admin/startBASMSync', + async () => { + const report = await basmEngine.startBASMSync() + return { status: 'success', message: 'BASM sync started and completed', data: report } + }, + checkAdminAuth as any + ) /** * Admin route to evict expired unproven topic transactions. */ - registerJsonRoute('/admin/evictUnproven', async req => { - const { topic, thresholdBlocks } = req.body ?? {} - const report = await basmEngine.evictUnprovenTransactions({ - topic: typeof topic === 'string' ? topic : undefined, - thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined - }) - this.logger.log({ operation: 'overlay.unproven_eviction', outcome: 'ok', report }) - return { status: 'success', message: 'Unproven eviction completed', data: report } - }, checkAdminAuth as any) + registerJsonRoute( + '/admin/evictUnproven', + async req => { + const { topic, thresholdBlocks } = req.body ?? {} + const report = await basmEngine.evictUnprovenTransactions({ + topic: typeof topic === 'string' ? topic : undefined, + thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined + }) + this.logger.log({ operation: 'overlay.unproven_eviction', outcome: 'ok', report }) + return { status: 'success', message: 'Unproven eviction completed', data: report } + }, + checkAdminAuth as any + ) /** * Admin route to refresh proofs for expired unproven topic transactions * using the configured proof providers. */ - registerJsonRoute('/admin/refreshUnprovenProofs', async req => { - const { topic, thresholdBlocks } = req.body ?? {} - const report = await basmEngine.refreshUnprovenTransactionProofs({ - topic: typeof topic === 'string' ? topic : undefined, - thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined, - proofProvider: async txid => await this.fetchConfiguredMerkleProof(txid) - }) - this.logger.log({ operation: 'overlay.unproven_proof_refresh', outcome: 'ok', report }) - return { status: 'success', message: 'Unproven proof refresh completed', data: report } - }, checkAdminAuth as any) + registerJsonRoute( + '/admin/refreshUnprovenProofs', + async req => { + const { topic, thresholdBlocks } = req.body ?? {} + const report = await basmEngine.refreshUnprovenTransactionProofs({ + topic: typeof topic === 'string' ? topic : undefined, + thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined, + proofProvider: async txid => await this.fetchConfiguredMerkleProof(txid) + }) + this.logger.log({ operation: 'overlay.unproven_proof_refresh', outcome: 'ok', report }) + return { status: 'success', message: 'Unproven proof refresh completed', data: report } + }, + checkAdminAuth as any + ) /** * Admin route to refresh proofs first, then evict rows that remain unproven. */ - registerJsonRoute('/admin/maintainUnproven', async req => { - const { topic, thresholdBlocks } = req.body ?? {} - const report = await basmEngine.maintainUnprovenTransactions({ - topic: typeof topic === 'string' ? topic : undefined, - thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined, - proofProvider: async txid => await this.fetchConfiguredMerkleProof(txid) - }) - this.logger.log({ operation: 'overlay.unproven_maintenance', outcome: 'ok', report }) - return { status: 'success', message: 'Unproven maintenance completed', data: report } - }, checkAdminAuth as any) + registerJsonRoute( + '/admin/maintainUnproven', + async req => { + const { topic, thresholdBlocks } = req.body ?? {} + const report = await basmEngine.maintainUnprovenTransactions({ + topic: typeof topic === 'string' ? topic : undefined, + thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined, + proofProvider: async txid => await this.fetchConfiguredMerkleProof(txid) + }) + this.logger.log({ operation: 'overlay.unproven_maintenance', outcome: 'ok', report }) + return { status: 'success', message: 'Unproven maintenance completed', data: report } + }, + checkAdminAuth as any + ) /** * Admin route to evict an outpoint, either from all services or a specific one. */ this.app.post('/admin/evictOutpoint', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { if (typeof req.body.service === 'string') { const service = engine.lookupServices[req.body.service] @@ -2482,11 +2855,13 @@ export default class OverlayExpress { * Admin route to run the janitor service with enhanced reporting. */ this.app.post('/admin/janitor', checkAdminAuth as any, (req, res) => { - ; (async () => { + ;(async () => { try { const janitor = this.createJanitor() const report: JanitorReport = await janitor.run() - return res.status(200).json({ status: 'success', message: 'Janitor run completed', data: report }) + return res + .status(200) + .json({ status: 'success', message: 'Janitor run completed', data: report }) } catch (error) { console.error(chalk.red('Error in /admin/janitor:'), error) return res.status(400).json({ @@ -2524,13 +2899,11 @@ export default class OverlayExpress { // Start listening on the configured port this.server = this.app.listen(this.port, () => { this.isListening = true - this.logger.log(chalk.green.bold(`${this.name} is ready and listening on local port ${this.port}`)) + this.logger.log( + chalk.green.bold(`${this.name} is ready and listening on local port ${this.port}`) + ) }) - configureHttpServer( - this.server, - edgePolicy.environmentPrefix, - edgePolicy.http - ) + configureHttpServer(this.server, edgePolicy.environmentPrefix, edgePolicy.http) } /** @@ -2539,12 +2912,12 @@ export default class OverlayExpress { * The operation is idempotent so multiple signal handlers or embedding * runtimes can share shutdown ownership safely. */ - async close (): Promise { + async close(): Promise { this.closePromise ??= this.closeResources() await this.closePromise } - private async closeResources (): Promise { + private async closeResources(): Promise { this.isListening = false if (this.basmBlockPollTimer !== undefined) { @@ -2560,14 +2933,15 @@ export default class OverlayExpress { const server = this.server this.server = undefined - const closeServer = server === undefined - ? Promise.resolve() - : new Promise((resolve, reject) => { - server.close(error => { - if (error !== undefined) reject(error) - else resolve() + const closeServer = + server === undefined + ? Promise.resolve() + : new Promise((resolve, reject) => { + server.close(error => { + if (error !== undefined) reject(error) + else resolve() + }) }) - }) await closeServer await Promise.all([ @@ -2583,7 +2957,7 @@ export default class OverlayExpress { * Runs the post-listen startup work: advertiser init, advertisement sync, * and the optional GASP/BASM background syncs. */ - private async runStartupSync (): Promise { + private async runStartupSync(): Promise { // The legacy Ninja advertiser has a setLookupEngine method. if (this.engine?.advertiser instanceof DiscoveryServices.WalletAdvertiser) { this.logger.log( @@ -2613,7 +2987,7 @@ export default class OverlayExpress { } /** Attempt a GASP sync at startup when enabled. */ - private async runGaspStartupSync (): Promise { + private async runGaspStartupSync(): Promise { if (!this.enableGASPSync) { this.logger.log(chalk.yellow(`${this.name} will not sync because GASP has been disabled.`)) return @@ -2628,7 +3002,7 @@ export default class OverlayExpress { } /** Attempt a BASM sync at startup when enabled, then begin tip-following. */ - private async runBasmStartupSync (): Promise { + private async runBasmStartupSync(): Promise { if (!(this.enableBASMSync || this.engineConfig.enableBASMSync === true)) { return } @@ -2648,7 +3022,7 @@ export default class OverlayExpress { } /** Poll for new blocks to advance anchor chains and detect reorgs. */ - private startBASMBlockPolling (): void { + private startBASMBlockPolling(): void { if (this.basmBlockPollIntervalMs <= 0) { return } @@ -2662,7 +3036,7 @@ export default class OverlayExpress { } /** Real-time reorg reconciliation via the go-chaintracks (Arcade) reorg SSE. */ - private startBASMReorgStream (): void { + private startBASMReorgStream(): void { const reorgStreamUrl = this.engineConfig.reorgStreamUrl ?? this.reorgStreamUrl const reorgScanDepth = this.engineConfig.reorgScanDepth ?? this.reorgScanDepth if (reorgStreamUrl === undefined || reorgStreamUrl === '') { @@ -2671,8 +3045,12 @@ export default class OverlayExpress { const basmEngine = this.engine as BASMCapableEngine | undefined this.reorgAdapter = new ReorgSseAdapter({ url: reorgStreamUrl, - onReorg: async input => { await basmEngine?.handleReorg(input) }, - onConnect: async () => { await basmEngine?.revalidateRecentAnchors(reorgScanDepth) }, + onReorg: async input => { + await basmEngine?.handleReorg(input) + }, + onConnect: async () => { + await basmEngine?.revalidateRecentAnchors(reorgScanDepth) + }, logger: this.logger }) this.reorgAdapter.start() @@ -2680,7 +3058,7 @@ export default class OverlayExpress { } /** Extend every topic's BASM anchor chain to the current chain tip. */ - private async advanceBASMAnchorChains (): Promise { + private async advanceBASMAnchorChains(): Promise { try { await (this.engine as BASMCapableEngine | undefined)?.advanceTopicAnchorChains() } catch (e) { @@ -2689,7 +3067,7 @@ export default class OverlayExpress { } /** Revalidate recent BASM anchors against the chain tracker, reconciling any reorg. */ - private async revalidateBASMAnchors (): Promise { + private async revalidateBASMAnchors(): Promise { try { const depth = this.engineConfig.reorgScanDepth ?? this.reorgScanDepth await (this.engine as BASMCapableEngine | undefined)?.revalidateRecentAnchors(depth) @@ -2698,14 +3076,18 @@ export default class OverlayExpress { } } - private startUnprovenMaintenance (): void { - const intervalMs = this.engineConfig.unprovenMaintenanceIntervalMs ?? this.unprovenMaintenanceIntervalMs + private startUnprovenMaintenance(): void { + const intervalMs = + this.engineConfig.unprovenMaintenanceIntervalMs ?? this.unprovenMaintenanceIntervalMs if (intervalMs <= 0) return const run = (): void => { void (async () => { try { - const report = await (this.engine as BASMCapableEngine | undefined)?.maintainUnprovenTransactions({ - thresholdBlocks: this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks, + const report = await ( + this.engine as BASMCapableEngine | undefined + )?.maintainUnprovenTransactions({ + thresholdBlocks: + this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks, proofProvider: async txid => await this.fetchConfiguredMerkleProof(txid) }) this.logger.log(chalk.green('Unproven transaction maintenance complete'), report) diff --git a/packages/overlays/overlay-express/src/ResourceBoundedLookupWrapper.ts b/packages/overlays/overlay-express/src/ResourceBoundedLookupWrapper.ts new file mode 100644 index 000000000..587269b33 --- /dev/null +++ b/packages/overlays/overlay-express/src/ResourceBoundedLookupWrapper.ts @@ -0,0 +1,99 @@ +import { + LookupService, + LookupFormula, + AdmissionMode, + SpendNotificationMode, + OutputAdmittedByTopic, + OutputSpent, + LookupServiceMetaData +} from '@bsv/overlay' +import { LookupQuestion } from '@bsv/sdk' + +/** + * Pushes the engine lookup ceiling into the built-in SHIP/SLAP services so a + * remote lookup cannot materialize an unbounded MongoDB result before Engine + * gets a chance to enforce its response limit. + * + * One extra row is requested deliberately. Engine turns that row into a clear + * range error instead of returning a silently truncated discovery result. + */ +export class ResourceBoundedLookupWrapper implements LookupService { + readonly admissionMode: AdmissionMode + readonly spendNotificationMode: SpendNotificationMode + + constructor( + private readonly wrapped: LookupService, + private readonly maxLookupResults: number + ) { + if ( + maxLookupResults !== -1 && + (!Number.isSafeInteger(maxLookupResults) || maxLookupResults < 1) + ) { + throw new TypeError('maxLookupResults must be -1 or a positive safe integer') + } + this.admissionMode = wrapped.admissionMode + this.spendNotificationMode = wrapped.spendNotificationMode + } + + async outputAdmittedByTopic(payload: OutputAdmittedByTopic): Promise { + return await this.wrapped.outputAdmittedByTopic(payload) + } + + async outputSpent(payload: OutputSpent): Promise { + if (typeof this.wrapped.outputSpent === 'function') { + return await this.wrapped.outputSpent(payload) + } + } + + async outputNoLongerRetainedInHistory( + txid: string, + outputIndex: number, + topic: string + ): Promise { + if (typeof this.wrapped.outputNoLongerRetainedInHistory === 'function') { + return await this.wrapped.outputNoLongerRetainedInHistory(txid, outputIndex, topic) + } + } + + async outputEvicted(txid: string, outputIndex: number): Promise { + return await this.wrapped.outputEvicted(txid, outputIndex) + } + + async lookup(question: LookupQuestion): Promise { + return await this.wrapped.lookup(this.boundQuestion(question)) + } + + async getDocumentation(): Promise { + return await this.wrapped.getDocumentation() + } + + async getMetaData(): Promise { + return await this.wrapped.getMetaData() + } + + private boundQuestion(question: LookupQuestion): LookupQuestion { + if (this.maxLookupResults === -1) return question + + const probeLimit = this.maxLookupResults + 1 + if (question.query === 'findAll') { + return { ...question, query: { findAll: true, limit: probeLimit } } + } + if ( + typeof question.query !== 'object' || + question.query === null || + Array.isArray(question.query) + ) { + return question + } + + const query = question.query as Record + const requestedLimit = query.limit + if ( + requestedLimit === undefined || + (typeof requestedLimit === 'number' && requestedLimit > probeLimit) + ) { + return { ...question, query: { ...query, limit: probeLimit } } + } + return question + } +} diff --git a/packages/overlays/overlay-express/src/__tests__/BanService.test.ts b/packages/overlays/overlay-express/src/__tests__/BanService.test.ts index f2cd8ce79..82a8115d5 100644 --- a/packages/overlays/overlay-express/src/__tests__/BanService.test.ts +++ b/packages/overlays/overlay-express/src/__tests__/BanService.test.ts @@ -83,7 +83,9 @@ describe('BanService', () => { }) it('should reject non-string domain (NoSQL injection prevention)', async () => { - await expect(banService.banDomain({ $ne: '' } as any)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.banDomain({ $ne: '' } as any)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) @@ -98,7 +100,9 @@ describe('BanService', () => { }) it('should reject non-string domain', async () => { - await expect(banService.unbanDomain({ $ne: '' } as any)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.unbanDomain({ $ne: '' } as any)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) @@ -124,7 +128,9 @@ describe('BanService', () => { }) it('should reject non-string domain', async () => { - await expect(banService.isDomainBanned(123 as any)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.isDomainBanned(123 as any)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) @@ -164,7 +170,9 @@ describe('BanService', () => { }) it('should reject non-string txid', async () => { - await expect(banService.banOutpoint({ $ne: '' } as any, 0)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.banOutpoint({ $ne: '' } as any, 0)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) @@ -179,7 +187,9 @@ describe('BanService', () => { }) it('should reject non-string txid', async () => { - await expect(banService.unbanOutpoint(42 as any, 0)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.unbanOutpoint(42 as any, 0)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) @@ -205,7 +215,9 @@ describe('BanService', () => { }) it('should reject non-string txid', async () => { - await expect(banService.isOutpointBanned(null as any, 0)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.isOutpointBanned(null as any, 0)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) @@ -250,6 +262,30 @@ describe('BanService', () => { expect(mockCollection.find).toHaveBeenCalledWith({ type: 'outpoint' }) }) + + it('should apply bounded pagination to MongoDB', async () => { + const toArray = jest.fn().mockResolvedValue([]) + const limit = jest.fn().mockReturnValue({ toArray, limit: jest.fn(), skip: jest.fn() }) + const skip = jest.fn().mockReturnValue({ limit, toArray, skip: jest.fn() }) + mockCollection.find.mockReturnValue({ + sort: jest.fn().mockReturnValue({ skip, limit, toArray }) + }) + + await banService.listBans('domain', 25, 50) + + expect(skip).toHaveBeenCalledWith(50) + expect(limit).toHaveBeenCalledWith(25) + expect(toArray).toHaveBeenCalled() + }) + + it.each([ + [0, 0], + [1.5, 0], + [10, -1], + [10, 1.5] + ])('rejects invalid pagination limit=%s skip=%s', async (limit, skip) => { + await expect(banService.listBans(undefined, limit, skip)).rejects.toThrow(TypeError) + }) }) describe('removeBan', () => { @@ -272,11 +308,15 @@ describe('BanService', () => { }) it('should reject non-string type (NoSQL injection prevention)', async () => { - await expect(banService.removeBan({ $ne: '' } as any, 'value')).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.removeBan({ $ne: '' } as any, 'value')).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) it('should reject non-string value (NoSQL injection prevention)', async () => { - await expect(banService.removeBan('domain', { $ne: '' } as any)).rejects.toThrow('Invalid input: expected a string value') + await expect(banService.removeBan('domain', { $ne: '' } as any)).rejects.toThrow( + 'Invalid input: expected a string value' + ) }) }) diff --git a/packages/overlays/overlay-express/src/__tests__/JanitorService.test.ts b/packages/overlays/overlay-express/src/__tests__/JanitorService.test.ts index c5d5910ae..0523c1bf5 100644 --- a/packages/overlays/overlay-express/src/__tests__/JanitorService.test.ts +++ b/packages/overlays/overlay-express/src/__tests__/JanitorService.test.ts @@ -67,6 +67,21 @@ describe('JanitorService', () => { expect(janitor).toBeDefined() }) + + it.each([ + [{ batchSize: 0 }, 'batchSize'], + [{ batchSize: 1.5 }, 'batchSize'], + [{ maxReportResults: 0 }, 'maxReportResults'], + [{ maxReportResults: 1.5 }, 'maxReportResults'] + ])('rejects invalid bounded configuration %o', (limits, expectedField) => { + expect( + () => + new JanitorService({ + mongoDb: mockDb, + ...limits + }) + ).toThrow(expectedField) + }) }) describe('run', () => { @@ -85,7 +100,7 @@ describe('JanitorService', () => { }) it('should handle errors during health checks', async () => { - (mockDb as any).collection = jest.fn().mockImplementation(() => { + ;(mockDb as any).collection = jest.fn().mockImplementation(() => { throw new Error('Database error') }) @@ -108,6 +123,37 @@ describe('JanitorService', () => { expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('completed')) }) + + it('should process every streamed record while bounding retained report details', async () => { + const outputs = Array.from({ length: 5 }, (_, index) => ({ + _id: String(index), + txid: `tx-${index}`, + outputIndex: index, + domain: 'invalid host', + down: 0 + })) + mockCollection.find.mockReturnValue({ + batchSize: jest.fn().mockReturnThis(), + async *[Symbol.asyncIterator]() { + for (const output of outputs) yield output + } + }) + + const janitor = new JanitorService({ + mongoDb: mockDb, + logger: mockLogger, + batchSize: 2, + maxReportResults: 2 + }) + + const report = await janitor.run() + + expect(report.summary.totalChecked).toBe(10) + expect(report.shipResults).toHaveLength(2) + expect(report.slapResults).toHaveLength(2) + expect(report.resultsTruncated).toBe(true) + expect(mockCollection.updateOne).toHaveBeenCalledTimes(10) + }) }) describe('output processing', () => { @@ -295,10 +341,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: 1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: 1 } }) }) it('should reject localhost by default', async () => { @@ -433,29 +476,29 @@ describe('JanitorService', () => { expect(global.fetch).not.toHaveBeenCalled() }) - it.each([ - 'https://8.8.8.8', - 'https://[2606:4700:4700::1111]' - ])('allows public IP health targets %s', async target => { - ;(global.fetch as jest.Mock).mockResolvedValue({ - ok: true, - headers: { get: jest.fn().mockReturnValue(null) }, - status: 200, - json: jest.fn().mockResolvedValue({ status: 'ok' }) - }) - const janitor = new JanitorService({ - mongoDb: mockDb, - logger: mockLogger - }) - - const result = await janitor.checkHost(target) - - expect(result.healthy).toBe(true) - expect(global.fetch).toHaveBeenCalledWith( - expect.stringMatching(/\/health$/), - expect.objectContaining({ redirect: 'error' }) - ) - }) + it.each(['https://8.8.8.8', 'https://[2606:4700:4700::1111]'])( + 'allows public IP health targets %s', + async target => { + ;(global.fetch as jest.Mock).mockResolvedValue({ + ok: true, + headers: { get: jest.fn().mockReturnValue(null) }, + status: 200, + json: jest.fn().mockResolvedValue({ status: 'ok' }) + }) + const janitor = new JanitorService({ + mongoDb: mockDb, + logger: mockLogger + }) + + const result = await janitor.checkHost(target) + + expect(result.healthy).toBe(true) + expect(global.fetch).toHaveBeenCalledWith( + expect.stringMatching(/\/health$/), + expect.objectContaining({ redirect: 'error' }) + ) + } + ) }) describe('health check', () => { @@ -484,10 +527,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: -1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: -1 } }) }) it('should not decrement when already at 0', async () => { @@ -542,10 +582,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: 1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: 1 } }) }) it('should delete output when down count reaches threshold', async () => { @@ -602,10 +639,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: 1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: 1 } }) }) it('should handle fetch errors', async () => { @@ -630,10 +664,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: 1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: 1 } }) }) it('should handle invalid JSON response', async () => { @@ -661,10 +692,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: 1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: 1 } }) }) it('should verify health endpoint returns status: ok', async () => { @@ -692,10 +720,7 @@ describe('JanitorService', () => { await janitor.run() - expect(mockCollection.updateOne).toHaveBeenCalledWith( - { _id: '123' }, - { $inc: { down: 1 } } - ) + expect(mockCollection.updateOne).toHaveBeenCalledWith({ _id: '123' }, { $inc: { down: 1 } }) }) }) @@ -841,10 +866,7 @@ describe('JanitorService', () => { await janitor.checkHost('example.com') - expect(global.fetch).toHaveBeenCalledWith( - 'https://example.com/health', - expect.any(Object) - ) + expect(global.fetch).toHaveBeenCalledWith('https://example.com/health', expect.any(Object)) }) }) @@ -901,6 +923,37 @@ describe('JanitorService', () => { expect(result.ship).toEqual([]) expect(result.slap).toEqual([]) }) + + it('pushes bounded report reads into MongoDB and supports explicit unlimited reads', async () => { + const records = [{ txid: 'tx', outputIndex: 0, domain: 'https://node.example' }] + const boundedCursor: any = { + limit: jest.fn().mockReturnThis(), + toArray: jest.fn().mockResolvedValue(records) + } + mockCollection.find.mockReturnValue(boundedCursor) + const bounded = new JanitorService({ + mongoDb: mockDb, + logger: mockLogger, + maxReportResults: 2 + }) + + await bounded.getHealthStatus() + expect(boundedCursor.limit).toHaveBeenCalledWith(2) + + const unlimitedCursor = { + toArray: jest.fn().mockResolvedValue(records) + } + mockCollection.find.mockReturnValue(unlimitedCursor) + const unlimited = new JanitorService({ + mongoDb: mockDb, + logger: mockLogger, + maxReportResults: -1 + }) + + const result = await unlimited.getHealthStatus() + expect(result.ship).toHaveLength(1) + expect(result.slap).toHaveLength(1) + }) }) describe('ban service integration', () => { diff --git a/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts b/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts index 21fda0fa9..3e80b3d95 100644 --- a/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts +++ b/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts @@ -19,16 +19,19 @@ jest.mock('@bsv/auth-express-middleware', () => ({ /** Creates a mock MongoDB Db object with a collection stub that supports BanService */ function createMockDbValue(): Record { + const cursor: Record = {} + cursor.sort = jest.fn().mockReturnValue(cursor) + cursor.skip = jest.fn().mockReturnValue(cursor) + cursor.limit = jest.fn().mockReturnValue(cursor) + cursor.toArray = jest.fn().mockResolvedValue([{ domain: 'node.example', txid: '01' }]) const mockCollection = { createIndex: jest.fn().mockResolvedValue(undefined), - find: jest.fn().mockReturnValue({ - sort: jest.fn().mockReturnValue({ toArray: jest.fn().mockResolvedValue([]) }), - toArray: jest.fn().mockResolvedValue([]) - }), - findOne: jest.fn().mockResolvedValue(null), + find: jest.fn().mockReturnValue(cursor), + findOne: jest.fn().mockResolvedValue({ domain: 'node.example' }), updateOne: jest.fn().mockResolvedValue({}), deleteOne: jest.fn().mockResolvedValue({}), - countDocuments: jest.fn().mockResolvedValue(0) + deleteMany: jest.fn().mockResolvedValue({ deletedCount: 1 }), + countDocuments: jest.fn().mockResolvedValue(1) } return { collection: jest.fn().mockReturnValue(mockCollection), @@ -396,6 +399,16 @@ describe('OverlayExpress', () => { overlayExpress.configureChainTracker('scripts only') expect(overlayExpress.chainTracker).toBe('scripts only') }) + + it('applies default tracker and provider configuration', () => { + overlayExpress.configureChainTracker() + overlayExpress.configureArcade('https://arcade.example') + overlayExpress.configureChaintracks('https://chaintracks.example') + + expect(overlayExpress.chainTracker).toBeDefined() + expect(overlayExpress.arcadeUrl).toBe('https://arcade.example') + expect(overlayExpress.reorgStreamUrl).toContain('chaintracks.example') + }) }) describe('configureArcApiKey', () => { @@ -897,10 +910,26 @@ describe('OverlayExpress', () => { refreshUnprovenTransactionProofs: jest.fn().mockResolvedValue({}), // @ts-expect-error - Mock return values maintainUnprovenTransactions: jest.fn().mockResolvedValue({}), + provideTopicAnchorTip: jest.fn().mockResolvedValue({ height: 1 }), + provideTopicAnchorRange: jest.fn().mockResolvedValue([{ height: 1 }]), + provideAdmittedList: jest.fn().mockResolvedValue({ txids: [] }), + provideCompoundMerklePath: jest.fn().mockResolvedValue({ path: [] }), + provideRawTransactions: jest.fn().mockResolvedValue({ transactions: [] }), + startBASMSync: jest.fn().mockResolvedValue({ topics: 1 }), + evictUnprovenTransactions: jest.fn().mockResolvedValue({ evicted: 1 }), + advanceTopicAnchorChains: jest.fn().mockResolvedValue({ advanced: 1 }), + revalidateRecentAnchors: jest.fn().mockResolvedValue({ revalidated: 1 }), evictAppliedTransaction: jest .fn() .mockResolvedValue({ evictedTransactions: 1, evictedOutputs: 1 }), - lookupServices: {}, + lookupServices: { + ls_one: { + outputEvicted: jest.fn().mockResolvedValue(undefined) + }, + ls_two: { + outputEvicted: jest.fn().mockRejectedValue(new Error('best-effort failure')) + } + }, advertiser: { // @ts-expect-error - Mock return values init: jest.fn().mockResolvedValue(undefined) @@ -925,6 +954,55 @@ describe('OverlayExpress', () => { instance.knex = mockKnex }) + const flushRoute = async (): Promise => { + await new Promise(resolve => setImmediate(resolve)) + await new Promise(resolve => setImmediate(resolve)) + } + + const mockResponse = (): any => { + const res: any = {} + res.status = jest.fn().mockReturnValue(res) + res.json = jest.fn().mockReturnValue(res) + res.send = jest.fn().mockReturnValue(res) + res.set = jest.fn().mockReturnValue(res) + res.setHeader = jest.fn().mockReturnValue(res) + return res + } + + const startAndCaptureRoutes = async (): Promise<{ getSpy: any; postSpy: any }> => { + const getSpy = jest.spyOn(instance.app, 'get') + const postSpy = jest.spyOn(instance.app, 'post') + jest.spyOn(instance.app, 'listen').mockImplementation((port: any, callback: any) => { + callback() + return {} as any + }) + await instance.start() + return { getSpy, postSpy } + } + + const invokeCapturedRoute = async ( + spy: any, + path: string, + request: Record = {} + ): Promise => { + const route = spy.mock.calls.find((call: any[]) => call[0] === path) + expect(route).toBeDefined() + const handler = route[route.length - 1] + const res = mockResponse() + handler( + { + headers: {}, + query: {}, + body: {}, + ...request + }, + res, + jest.fn() + ) + await flushRoute() + return res + } + it('should throw if engine not configured', async () => { const freshInstance = new OverlayExpress('Test', 'key', 'example.com') const mockKnex = { @@ -1058,6 +1136,24 @@ describe('OverlayExpress', () => { } }) + it('returns the callback STEAK exactly once from /submit', async () => { + const callbackSteak = { status: 'success', txid: 'callback-txid' } + mockEngine.submit.mockImplementationOnce(async (_beef: any, callback: any) => { + callback(callbackSteak) + return { status: 'success', txid: 'returned-txid' } + }) + const { postSpy } = await startAndCaptureRoutes() + + const response = await invokeCapturedRoute(postSpy, '/submit', { + headers: { 'x-topics': 'tm_callback' }, + body: Buffer.from([1, 2, 3]) + }) + + expect(response.status).toHaveBeenCalledWith(200) + expect(response.json).toHaveBeenCalledTimes(1) + expect(response.json).toHaveBeenCalledWith(callbackSteak) + }) + it('returns a clean 400 for an empty /submit body', async () => { const postSpy = jest.spyOn(instance.app, 'post') jest.spyOn(instance.app, 'listen').mockImplementation((port: any, callback: any) => { @@ -1176,6 +1272,7 @@ describe('OverlayExpress', () => { expect(getSpy.mock.calls.find(call => call[0] === '/health')).toBeDefined() expect(getSpy.mock.calls.find(call => call[0] === '/health/live')).toBeDefined() expect(getSpy.mock.calls.find(call => call[0] === '/health/ready')).toBeDefined() + expect(getSpy.mock.calls.find(call => call[0] === '/healthz')).toBeDefined() }) it('should return detailed readiness health', async () => { @@ -1213,6 +1310,39 @@ describe('OverlayExpress', () => { ) }) + it('executes compatibility health probes and contains probe failures', async () => { + const loggerError = jest.spyOn(instance.logger, 'error').mockImplementation(() => {}) + const { getSpy } = await startAndCaptureRoutes() + + expect((await invokeCapturedRoute(getSpy, '/health/live')).status).toHaveBeenCalledWith(200) + expect((await invokeCapturedRoute(getSpy, '/healthz')).status).toHaveBeenCalledWith(200) + expect((await invokeCapturedRoute(getSpy, '/health')).status).toHaveBeenCalledWith(200) + + const collectHealthReport = jest.spyOn(instance as any, 'collectHealthReport') + collectHealthReport.mockResolvedValueOnce({ status: 'degraded', live: false }) + expect((await invokeCapturedRoute(getSpy, '/healthz')).status).toHaveBeenCalledWith(503) + + collectHealthReport.mockRejectedValueOnce(new Error('probe failed')) + const failed = await invokeCapturedRoute(getSpy, '/healthz') + expect(failed.status).toHaveBeenCalledWith(500) + expect(failed.json).toHaveBeenCalledWith({ + status: 'error', + message: 'Health report unavailable' + }) + expect(loggerError).toHaveBeenCalledWith( + expect.objectContaining({ operation: 'overlay.healthz' }) + ) + }) + + it('constructs the janitor with bounded operator defaults', () => { + const defaultJanitor = (instance as any).createJanitor() + instance.configureJanitor({ batchSize: 10, maxReportResults: 20 }) + const configuredJanitor = (instance as any).createJanitor() + + expect(defaultJanitor).toBeDefined() + expect(configuredJanitor).toBeDefined() + }) + it('should register admin routes', async () => { const postSpy = jest.spyOn(instance.app, 'post') jest.spyOn(instance.app, 'listen').mockImplementation((port: any, callback: any) => { @@ -1485,6 +1615,302 @@ describe('OverlayExpress', () => { }) }) + it('executes public discovery, GASP, and bounded BASM routes', async () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => {}) + const { getSpy, postSpy } = await startAndCaptureRoutes() + + expect((await invokeCapturedRoute(getSpy, '/')).send).toHaveBeenCalled() + expect((await invokeCapturedRoute(getSpy, '/listTopicManagers')).status).toHaveBeenCalledWith( + 200 + ) + expect( + (await invokeCapturedRoute(getSpy, '/listLookupServiceProviders')).status + ).toHaveBeenCalledWith(200) + expect( + ( + await invokeCapturedRoute(getSpy, '/getDocumentationForTopicManager', { + query: { manager: 'tm_test' } + }) + ).send + ).toHaveBeenCalledWith('# Docs') + expect( + ( + await invokeCapturedRoute(getSpy, '/getDocumentationForLookupServiceProvider', { + query: { lookupService: 'ls_test' } + }) + ).send + ).toHaveBeenCalledWith('# Docs') + + const lookup = await invokeCapturedRoute(postSpy, '/lookup', { + headers: {}, + body: { service: 'ls_test', query: { findAll: true } } + }) + expect(lookup.status).toHaveBeenCalledWith(200) + const invalidLookup = await invokeCapturedRoute(postSpy, '/lookup', { + headers: {}, + body: { query: {} } + }) + expect(invalidLookup.status).toHaveBeenCalledWith(400) + + await invokeCapturedRoute(postSpy, '/requestSyncResponse', { + headers: { 'x-bsv-topic': 'tm_test' }, + body: { since: 1 } + }) + await invokeCapturedRoute(postSpy, '/requestForeignGASPNode', { + body: { graphID: 'graph', txid: '01', outputIndex: 0 } + }) + expect(mockEngine.provideForeignSyncResponse).toHaveBeenCalledWith({ since: 1 }, 'tm_test') + expect(mockEngine.provideForeignGASPNode).toHaveBeenCalledWith('graph', '01', 0) + + const topicRequest = { headers: { 'x-bsv-topic': 'tm_test' } } + await invokeCapturedRoute(postSpy, '/requestTopicAnchorTip', topicRequest) + await invokeCapturedRoute(postSpy, '/requestTopicAnchorRange', { + ...topicRequest, + body: { fromHeight: 1, toHeight: 3 } + }) + await invokeCapturedRoute(postSpy, '/requestAdmittedList', { + ...topicRequest, + body: { blockHeight: 2, blockHash: 'hash' } + }) + await invokeCapturedRoute(postSpy, '/requestCompoundMerklePath', { + ...topicRequest, + body: { blockHeight: 2, txids: ['01', '02'] } + }) + await invokeCapturedRoute(postSpy, '/requestRawTransactions', { + body: { txids: ['01'] } + }) + expect(mockEngine.provideTopicAnchorTip).toHaveBeenCalledWith('tm_test') + expect(mockEngine.provideTopicAnchorRange).toHaveBeenCalledWith('tm_test', 1, 3) + expect(mockEngine.provideAdmittedList).toHaveBeenCalledWith('tm_test', 2, 'hash') + expect(mockEngine.provideCompoundMerklePath).toHaveBeenCalledWith('tm_test', 2, ['01', '02']) + expect(mockEngine.provideRawTransactions).toHaveBeenCalledWith(['01']) + + for (const [path, request] of [ + ['/requestTopicAnchorTip', { headers: {} }], + ['/requestTopicAnchorRange', { ...topicRequest, body: { fromHeight: 4, toHeight: 3 } }], + ['/requestTopicAnchorRange', { ...topicRequest, body: { fromHeight: 0, toHeight: 1000 } }], + ['/requestCompoundMerklePath', { ...topicRequest, body: { txids: [1] } }], + ['/requestRawTransactions', { body: { txids: 'not-an-array' } }], + ['/requestRawTransactions', { body: { txids: Array(1001).fill('01') } }] + ] as const) { + const response = await invokeCapturedRoute(postSpy, path, request) + expect(response.status).toHaveBeenCalledWith(400) + } + + consoleError.mockRestore() + }) + + it('enforces admin authentication and executes bounded record and ban operations', async () => { + instance.configureAdminIdentityKey('admin-identity') + const banService = { + getStats: jest + .fn() + .mockResolvedValue({ domainBans: 1, outpointBans: 1, totalBans: 2 }), + banDomain: jest.fn().mockResolvedValue(undefined), + banOutpoint: jest.fn().mockResolvedValue(undefined), + removeBan: jest.fn().mockResolvedValue(undefined), + listBans: jest.fn().mockResolvedValue([{ type: 'domain', value: 'node.example' }]) + } + instance.banService = banService as any + const janitor = { + checkHost: jest.fn().mockResolvedValue({ ok: true, responseTime: 10 }), + run: jest.fn().mockResolvedValue({ checked: 1, removed: 0 }) + } + jest.spyOn(instance as any, 'createJanitor').mockReturnValue(janitor) + const { getSpy, postSpy } = await startAndCaptureRoutes() + + const statsRoute = getSpy.mock.calls.find((call: any[]) => call[0] === '/admin/stats') + expect(statsRoute).toBeDefined() + const checkAdminAuth = statsRoute[1] + const next = jest.fn() + checkAdminAuth({ headers: {}, auth: { identityKey: 'admin-identity' } }, mockResponse(), next) + checkAdminAuth( + { headers: { authorization: `Bearer ${instance.getAdminToken()}` } }, + mockResponse(), + next + ) + expect(next).toHaveBeenCalledTimes(2) + + const invalidCredentials = mockResponse() + checkAdminAuth( + { headers: { authorization: 'Bearer invalid-token' } }, + invalidCredentials, + next + ) + expect(invalidCredentials.status).toHaveBeenCalledWith(403) + const missingCredentials = mockResponse() + checkAdminAuth({ headers: {} }, missingCredentials, next) + expect(missingCredentials.status).toHaveBeenCalledWith(401) + + expect((await invokeCapturedRoute(getSpy, '/admin/config')).status).toHaveBeenCalledWith(200) + expect((await invokeCapturedRoute(getSpy, '/admin/stats')).status).toHaveBeenCalledWith(200) + expect( + ( + await invokeCapturedRoute(getSpy, '/admin/ship-records', { + query: { search: 'node', page: '2', limit: '2' } + }) + ).status + ).toHaveBeenCalledWith(200) + expect( + ( + await invokeCapturedRoute(getSpy, '/admin/slap-records', { + query: { page: '1', limit: 'unlimited' } + }) + ).status + ).toHaveBeenCalledWith(200) + expect( + ( + await invokeCapturedRoute(getSpy, '/admin/ship-records', { + query: { page: 2, limit: 2 } + }) + ).status + ).toHaveBeenCalledWith(200) + + for (const [path, query] of [ + ['/admin/ship-records', { page: '1', limit: '0' }], + ['/admin/slap-records', { page: '1000000', limit: '200' }] + ] as const) { + expect((await invokeCapturedRoute(getSpy, path, { query })).status).toHaveBeenCalledWith( + 400 + ) + } + expect( + ( + await invokeCapturedRoute(postSpy, '/admin/health-check', { + body: { url: 'https://node.example/healthz' } + }) + ).status + ).toHaveBeenCalledWith(200) + + await invokeCapturedRoute(postSpy, '/admin/ban', { + body: { type: 'domain', value: 'node.example', reason: 'operator request' } + }) + await invokeCapturedRoute(postSpy, '/admin/ban', { + body: { type: 'outpoint', value: 'abcd.2', reason: 'operator request' } + }) + await invokeCapturedRoute(postSpy, '/admin/unban', { + body: { type: 'domain', value: 'node.example' } + }) + const bans = await invokeCapturedRoute(getSpy, '/admin/bans', { + query: { type: 'domain', page: '1', limit: '5' } + }) + expect(bans.status).toHaveBeenCalledWith(200) + await invokeCapturedRoute(postSpy, '/admin/remove-token', { + body: { + txid: 'abcd', + outputIndex: 2, + ban: true, + banDomain: true + } + }) + + instance.banService = undefined + expect( + ( + await invokeCapturedRoute(postSpy, '/admin/ban', { + body: { type: 'domain', value: 'unavailable.example' } + }) + ).status + ).toHaveBeenCalledWith(400) + instance.banService = banService as any + for (const [path, body] of [ + ['/admin/ban', { type: 'invalid', value: 'node.example' }], + ['/admin/unban', { type: 'invalid', value: 'node.example' }], + ['/admin/remove-token', { txid: 42, outputIndex: 'invalid' }] + ] as const) { + expect((await invokeCapturedRoute(postSpy, path, { body })).status).toHaveBeenCalledWith( + 400 + ) + } + + expect(banService.banDomain).toHaveBeenCalled() + expect(banService.banOutpoint).toHaveBeenCalled() + expect(banService.removeBan).toHaveBeenCalledWith('domain', 'node.example') + expect(banService.listBans).toHaveBeenCalledWith('domain', 5, 0) + expect(mockEngine.lookupServices.ls_one.outputEvicted).toHaveBeenCalled() + expect(janitor.checkHost).toHaveBeenCalledWith('https://node.example/healthz') + }) + + it('honors operator-unlimited admin pagination without applying cursor limits', async () => { + const environment = { + OVERLAY_ADMIN_LIST_DEFAULT_LIMIT: process.env.OVERLAY_ADMIN_LIST_DEFAULT_LIMIT, + OVERLAY_ADMIN_LIST_MAX_LIMIT: process.env.OVERLAY_ADMIN_LIST_MAX_LIMIT, + OVERLAY_ADMIN_LIST_MAX_OFFSET: process.env.OVERLAY_ADMIN_LIST_MAX_OFFSET + } + process.env.OVERLAY_ADMIN_LIST_DEFAULT_LIMIT = '-1' + process.env.OVERLAY_ADMIN_LIST_MAX_LIMIT = '-1' + process.env.OVERLAY_ADMIN_LIST_MAX_OFFSET = '-1' + try { + const { getSpy } = await startAndCaptureRoutes() + + for (const path of ['/admin/ship-records', '/admin/slap-records']) { + const response = await invokeCapturedRoute(getSpy, path, { + query: { page: '999', limit: 'unlimited' } + }) + expect(response.status).toHaveBeenCalledWith(200) + expect(response.json).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ page: 1, limit: -1, pages: 1 }) + }) + ) + } + } finally { + for (const [key, value] of Object.entries(environment)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } + }) + + it('executes authenticated sync, maintenance, eviction, and janitor operations', async () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => {}) + const janitor = { + checkHost: jest.fn().mockResolvedValue({ ok: true }), + run: jest.fn().mockResolvedValue({ checked: 2, removed: 1 }) + } + jest.spyOn(instance as any, 'createJanitor').mockReturnValue(janitor) + mockEngine.refreshUnprovenTransactionProofs.mockImplementationOnce(async (options: any) => { + await options.proofProvider('01') + return { refreshed: 1 } + }) + mockEngine.maintainUnprovenTransactions.mockImplementationOnce(async (options: any) => { + await options.proofProvider('02') + return { maintained: 1 } + }) + const { postSpy } = await startAndCaptureRoutes() + + const successfulRoutes: Array<[string, Record]> = [ + ['/admin/syncAdvertisements', {}], + ['/admin/startGASPSync', {}], + ['/admin/startBASMSync', {}], + ['/admin/evictUnproven', { body: { topic: 'tm_test', thresholdBlocks: 12 } }], + ['/admin/refreshUnprovenProofs', { body: { topic: 'tm_test', thresholdBlocks: 12 } }], + ['/admin/maintainUnproven', { body: { topic: 'tm_test', thresholdBlocks: 12 } }], + ['/admin/evictOutpoint', { body: { service: 'ls_one', txid: 'abcd', outputIndex: 2 } }], + ['/admin/janitor', {}] + ] + for (const [path, request] of successfulRoutes) { + const response = await invokeCapturedRoute(postSpy, path, request) + expect(response.status).toHaveBeenCalledWith(200) + } + + expect(mockEngine.syncAdvertisements).toHaveBeenCalled() + expect(mockEngine.startGASPSync).toHaveBeenCalled() + expect(mockEngine.startBASMSync).toHaveBeenCalled() + expect(mockEngine.evictUnprovenTransactions).toHaveBeenCalledWith({ + topic: 'tm_test', + thresholdBlocks: 12 + }) + expect(mockEngine.refreshUnprovenTransactionProofs).toHaveBeenCalledWith( + expect.objectContaining({ topic: 'tm_test', thresholdBlocks: 12 }) + ) + expect(mockEngine.maintainUnprovenTransactions).toHaveBeenCalledWith( + expect.objectContaining({ topic: 'tm_test', thresholdBlocks: 12 }) + ) + expect(janitor.run).toHaveBeenCalled() + consoleError.mockRestore() + }) + it('should run knex migrations on start', async () => { jest.spyOn(instance.app, 'listen').mockImplementation((port: any, callback: any) => { callback() diff --git a/packages/overlays/overlay-express/src/__tests__/ResourceBoundedLookupWrapper.test.ts b/packages/overlays/overlay-express/src/__tests__/ResourceBoundedLookupWrapper.test.ts new file mode 100644 index 000000000..f775a3537 --- /dev/null +++ b/packages/overlays/overlay-express/src/__tests__/ResourceBoundedLookupWrapper.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect, jest } from '@jest/globals' +import { LookupService } from '@bsv/overlay' +import { ResourceBoundedLookupWrapper } from '../ResourceBoundedLookupWrapper.js' + +const makeService = (): jest.Mocked => + ({ + admissionMode: 'locking-script', + spendNotificationMode: 'none', + outputAdmittedByTopic: jest.fn().mockResolvedValue(undefined), + outputSpent: jest.fn().mockResolvedValue(undefined), + outputNoLongerRetainedInHistory: jest.fn().mockResolvedValue(undefined), + outputEvicted: jest.fn().mockResolvedValue(undefined), + lookup: jest.fn().mockResolvedValue([]), + getDocumentation: jest.fn().mockResolvedValue('docs'), + getMetaData: jest.fn().mockResolvedValue({ name: 'test', shortDescription: 'test' }) + }) as any + +describe('ResourceBoundedLookupWrapper', () => { + it('turns the legacy findAll query into a bounded overflow probe', async () => { + const service = makeService() + const wrapper = new ResourceBoundedLookupWrapper(service, 1000) + + await wrapper.lookup({ service: 'ls_ship', query: 'findAll' }) + + expect(service.lookup).toHaveBeenCalledWith({ + service: 'ls_ship', + query: { findAll: true, limit: 1001 } + }) + }) + + it('adds a bound to filtered queries that omit a limit', async () => { + const service = makeService() + const wrapper = new ResourceBoundedLookupWrapper(service, 50) + + await wrapper.lookup({ service: 'ls_slap', query: { service: 'message-box' } }) + + expect(service.lookup).toHaveBeenCalledWith({ + service: 'ls_slap', + query: { service: 'message-box', limit: 51 } + }) + }) + + it('caps an explicitly oversized request but preserves smaller pages', async () => { + const service = makeService() + const wrapper = new ResourceBoundedLookupWrapper(service, 10) + + await wrapper.lookup({ service: 'ls_ship', query: { findAll: true, limit: 100 } }) + await wrapper.lookup({ service: 'ls_ship', query: { findAll: true, limit: 4 } }) + + expect(service.lookup).toHaveBeenNthCalledWith(1, { + service: 'ls_ship', + query: { findAll: true, limit: 11 } + }) + expect(service.lookup).toHaveBeenNthCalledWith(2, { + service: 'ls_ship', + query: { findAll: true, limit: 4 } + }) + }) + + it('preserves all lookup questions when the operator selects unlimited', async () => { + const service = makeService() + const wrapper = new ResourceBoundedLookupWrapper(service, -1) + const question = { service: 'ls_ship', query: 'findAll' } as const + + await wrapper.lookup(question) + + expect(service.lookup).toHaveBeenCalledWith(question) + }) + + it('rejects invalid resource limits', () => { + expect(() => new ResourceBoundedLookupWrapper(makeService(), 0)).toThrow(TypeError) + }) + + it('delegates lifecycle notifications and service metadata', async () => { + const service = makeService() + const wrapper = new ResourceBoundedLookupWrapper(service, 10) + const admitted = { txid: '01', outputIndex: 0, topic: 'tm_test' } as any + const spent = { txid: '01', outputIndex: 0, topic: 'tm_test' } as any + + await wrapper.outputAdmittedByTopic(admitted) + await wrapper.outputSpent(spent) + await wrapper.outputNoLongerRetainedInHistory('01', 0, 'tm_test') + await wrapper.outputEvicted('01', 0) + + expect(service.outputAdmittedByTopic).toHaveBeenCalledWith(admitted) + expect(service.outputSpent).toHaveBeenCalledWith(spent) + expect(service.outputNoLongerRetainedInHistory).toHaveBeenCalledWith('01', 0, 'tm_test') + expect(service.outputEvicted).toHaveBeenCalledWith('01', 0) + await expect(wrapper.getDocumentation()).resolves.toBe('docs') + await expect(wrapper.getMetaData()).resolves.toEqual({ name: 'test', shortDescription: 'test' }) + }) + + it('supports legacy services with optional notification hooks omitted', async () => { + const service = makeService() + delete (service as any).outputSpent + delete (service as any).outputNoLongerRetainedInHistory + const wrapper = new ResourceBoundedLookupWrapper(service, 10) + + await expect(wrapper.outputSpent({} as any)).resolves.toBeUndefined() + await expect( + wrapper.outputNoLongerRetainedInHistory('01', 0, 'tm_test') + ).resolves.toBeUndefined() + await expect( + wrapper.lookup({ service: 'ls_test', query: 'custom-scalar-query' } as any) + ).resolves.toEqual([]) + }) +}) diff --git a/packages/overlays/overlay-express/src/security/edgePolicy.test.ts b/packages/overlays/overlay-express/src/security/edgePolicy.test.ts index 1e169aee3..3d59d3882 100644 --- a/packages/overlays/overlay-express/src/security/edgePolicy.test.ts +++ b/packages/overlays/overlay-express/src/security/edgePolicy.test.ts @@ -5,13 +5,18 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readAllowedOrigins, readBodyLimitBytes, readCorsOriginSetting, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders } from './edgePolicy' -async function listen (app: express.Express): Promise<{ +async function listen(app: express.Express): Promise<{ server: Server origin: string }> { @@ -25,7 +30,7 @@ async function listen (app: express.Express): Promise<{ return { server, origin: `http://127.0.0.1:${address.port}` } } -async function close (server: Server): Promise { +async function close(server: Server): Promise { await new Promise((resolve, reject) => { server.close(error => { if (error != null) reject(error) @@ -47,10 +52,12 @@ describe('shared service edge policy', () => { delete process.env.CORS_MODE delete process.env.CORS_ALLOWED_ORIGINS const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -78,8 +85,9 @@ describe('shared service edge policy', () => { }) expect(preflight.status).toBe(204) expect(preflight.headers.get('access-control-allow-origin')).toBe('*') - expect(preflight.headers.get('access-control-allow-headers')) - .toContain('X-BSV-Action-Batch-Encoding') + expect(preflight.headers.get('access-control-allow-headers')).toContain( + 'X-BSV-Action-Batch-Encoding' + ) } finally { await close(server) } @@ -88,10 +96,12 @@ describe('shared service edge policy', () => { it('allows only explicitly configured browser origins', async () => { process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -124,10 +134,12 @@ describe('shared service edge policy', () => { delete process.env.TEST_CORS_ALLOWED_ORIGINS delete process.env.CORS_ALLOWED_ORIGINS const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -146,10 +158,12 @@ describe('shared service edge policy', () => { it('answers allowed preflight without wildcard policy', async () => { process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) const { server, origin } = await listen(app) try { @@ -170,16 +184,20 @@ describe('shared service edge policy', () => { it('rejects wildcard and malformed origin configuration', () => { process.env.TEST_CORS_ALLOWED_ORIGINS = '*' - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })).toThrow(/wildcard/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ).toThrow(/wildcard/) process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example/path' - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })).toThrow(/without paths/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ).toThrow(/without paths/) }) it('validates the complete CORS mode configuration matrix', () => { @@ -202,14 +220,8 @@ describe('shared service edge policy', () => { process.env.TEST_CORS_MODE = 'allowlist' process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example, https://wallet.example, https://wui.example' - expect(readAllowedOrigins('TEST')).toEqual([ - 'https://wallet.example', - 'https://wui.example' - ]) - expect(readCorsOriginSetting('TEST')).toEqual([ - 'https://wallet.example', - 'https://wui.example' - ]) + expect(readAllowedOrigins('TEST')).toEqual(['https://wallet.example', 'https://wui.example']) + expect(readCorsOriginSetting('TEST')).toEqual(['https://wallet.example', 'https://wui.example']) delete process.env.TEST_CORS_MODE delete process.env.TEST_CORS_ALLOWED_ORIGINS @@ -217,21 +229,27 @@ describe('shared service edge policy', () => { }) it('validates explicit origin and credential options', () => { - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowedOrigins: ['null'] - })).toThrow(/opaque/) - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowedOrigins: ['not an origin'] - })).toThrow(/invalid origin/) - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowCredentials: true - })).toThrow(/cookie credentials/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowedOrigins: ['null'] + }) + ).toThrow(/opaque/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowedOrigins: ['not an origin'] + }) + ).toThrow(/invalid origin/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowCredentials: true + }) + ).toThrow(/cookie credentials/) const disabled = corsPolicy({ environmentPrefix: 'TEST', @@ -277,11 +295,7 @@ describe('shared service edge policy', () => { sendStatus: jest.fn() } const next = jest.fn() - middleware( - { get: () => 'https://wallet.example', method: 'GET' } as any, - response as any, - next - ) + middleware({ get: () => 'https://wallet.example', method: 'GET' } as any, response as any, next) expect(headers.get('Vary')).toBe('Accept-Encoding, Origin') expect(headers.get('Access-Control-Allow-Origin')).toBe('https://wallet.example') expect(headers.get('Access-Control-Allow-Credentials')).toBe('true') @@ -296,10 +310,12 @@ describe('shared service edge policy', () => { process.env.TEST_CORS_MODE = 'allowlist' process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -342,15 +358,17 @@ describe('shared service edge policy', () => { process.env.TEST_STRICT_TRANSPORT_SECURITY = 'false' const app = express() app.enable('trust proxy') - app.use(securityHeaders({ - environmentPrefix: 'TEST', - contentSecurityPolicy: "default-src 'none'", - crossOriginResourcePolicy: 'same-origin', - crossOriginOpenerPolicy: 'same-origin', - frameOptions: 'DENY', - permissionsPolicy: 'camera=()', - strictTransportSecurity: true - })) + app.use( + securityHeaders({ + environmentPrefix: 'TEST', + contentSecurityPolicy: "default-src 'none'", + crossOriginResourcePolicy: 'same-origin', + crossOriginOpenerPolicy: 'same-origin', + frameOptions: 'DENY', + permissionsPolicy: 'camera=()', + strictTransportSecurity: true + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -375,10 +393,12 @@ describe('shared service edge policy', () => { delete process.env.TEST_CORS_ALLOWED_ORIGINS const app = express() app.use(securityHeaders({ environmentPrefix: 'TEST' })) - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -389,8 +409,9 @@ describe('shared service edge policy', () => { expect(response.status).toBe(200) expect(response.headers.get('access-control-allow-origin')).toBe('*') expect(response.headers.get('access-control-allow-credentials')).toBeNull() - expect(response.headers.get('content-security-policy')) - .toBe("default-src 'self'; connect-src https:") + expect(response.headers.get('content-security-policy')).toBe( + "default-src 'self'; connect-src https:" + ) } finally { await close(server) } @@ -519,7 +540,9 @@ describe('shared service edge policy', () => { app.use(concurrencyLimit('TEST', 10)) let releaseFirst: (() => void) | undefined app.get('/', async (_req, res) => { - await new Promise(resolve => { releaseFirst = resolve }) + await new Promise(resolve => { + releaseFirst = resolve + }) res.json({ ok: true }) }) const { server, origin } = await listen(app) @@ -528,7 +551,8 @@ describe('shared service edge policy', () => { headersTimeoutMs: 10_000, keepAliveTimeoutMs: 5_000, socketTimeoutMs: 30_000, - maxRequestsPerSocket: 100 + maxRequestsPerSocket: 100, + maxConnections: 50 }) try { @@ -545,8 +569,156 @@ describe('shared service edge policy', () => { expect(server.headersTimeout).toBe(10_000) expect(server.keepAliveTimeout).toBe(5_000) expect(server.maxRequestsPerSocket).toBe(100) + expect(server.maxConnections).toBe(50) } finally { await close(server) } }) + + it('honors explicit unlimited body, response, concurrency, and connection limits', () => { + process.env.TEST_MAX_BODY_BYTES = '-1' + expect(readBodyLimitBytes('TEST', 256)).toBe(Number.MAX_SAFE_INTEGER) + + process.env.TEST_MAX_RESPONSE_BYTES = 'unlimited' + const responseNext = jest.fn() + responseSizeLimit('TEST', 256)({} as any, {} as any, responseNext) + expect(responseNext).toHaveBeenCalledTimes(1) + + process.env.TEST_MAX_CONCURRENT_REQUESTS = '-1' + const concurrencyNext = jest.fn() + concurrencyLimit('TEST', 8)({} as any, {} as any, concurrencyNext) + expect(concurrencyNext).toHaveBeenCalledTimes(1) + + process.env.TEST_MAX_CONNECTIONS = '-1' + const server = { + setTimeout: jest.fn() + } as unknown as Server + configureHttpServer(server, 'TEST', { + requestTimeoutMs: 30_000, + headersTimeoutMs: 10_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 30_000, + maxRequestsPerSocket: 100, + maxConnections: 50 + }) + expect(server.maxConnections).toBe(Number.MAX_SAFE_INTEGER) + }) + + it('rejects invalid positive HTTP server settings', () => { + process.env.TEST_REQUEST_TIMEOUT_MS = '0' + const server = { + setTimeout: jest.fn() + } as unknown as Server + + expect(() => + configureHttpServer(server, 'TEST', { + requestTimeoutMs: 30_000, + headersTimeoutMs: 10_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 30_000, + maxRequestsPerSocket: 100, + maxConnections: 50 + }) + ).toThrow(/positive integer/) + }) + + it('selects tested resource profiles and explicit operator limits', () => { + expect(readResourceProfile('TEST')).toBe('standard') + process.env.TEST_RESOURCE_PROFILE = 'high-throughput' + expect(readResourceProfile('TEST')).toBe('high-throughput') + expect(profileValue('small', { small: 1, standard: 2, highThroughput: 3 })).toBe(1) + expect(profileValue('standard', { small: 1, standard: 2, highThroughput: 3 })).toBe(2) + expect(profileValue('high-throughput', { small: 1, standard: 2, highThroughput: 3 })).toBe(3) + + process.env.TEST_MAX_ITEMS = '1000' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(1_000) + process.env.TEST_MAX_ITEMS = '-1' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(-1) + process.env.TEST_MAX_ITEMS = 'unlimited' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(-1) + process.env.TEST_MAX_ITEMS = '0' + expect(() => readResourceLimit('TEST', 'MAX_ITEMS', 100)).toThrow(/positive integer/) + + process.env.TEST_RESOURCE_PROFILE = 'oversized' + expect(() => readResourceProfile('TEST')).toThrow(/small, standard, or high-throughput/) + }) + + it('tolerates only repeated initial slashes for compatibility', () => { + const next = jest.fn() + const request = { url: '///auth/start?mode=test' } + initialDoubleSlashCompatibility(request as any, {} as any, next) + expect(request.url).toBe('/auth/start?mode=test') + expect(next).toHaveBeenCalledTimes(1) + + const interior = { url: '/auth//start' } + initialDoubleSlashCompatibility(interior as any, {} as any, jest.fn()) + expect(interior.url).toBe('/auth//start') + }) + + it('rejects materialized responses above the configured byte budget', async () => { + process.env.TEST_MAX_RESPONSE_BYTES = '128' + const app = express() + app.use(responseSizeLimit('TEST', 1024)) + app.get('/small', (_req, res) => res.json({ ok: true })) + app.get('/large', (_req, res) => res.json({ value: 'x'.repeat(512) })) + const { server, origin } = await listen(app) + + try { + const small = await fetch(`${origin}/small`) + expect(small.status).toBe(200) + await expect(small.json()).resolves.toEqual({ ok: true }) + + const large = await fetch(`${origin}/large`) + expect(large.status).toBe(413) + await expect(large.json()).resolves.toMatchObject({ code: 'ERR_RESPONSE_TOO_LARGE' }) + } finally { + await close(server) + } + }) + + it.each([ + ['send', '12345', undefined], + ['send', Buffer.from('12345'), undefined], + ['send', new Uint8Array([1, 2, 3, 4, 5]), undefined], + ['send', { value: '12345' }, undefined], + ['end', '12345', 'utf8'], + ['end', Buffer.from('12345'), undefined], + ['end', new Uint8Array([1, 2, 3, 4, 5]), undefined], + ['end', { value: '12345' }, undefined] + ])('bounds every materialized %s response shape', (method, value, encoding) => { + process.env.TEST_MAX_RESPONSE_BYTES = '4' + let response: any + const originalEnd = jest.fn(() => response) + const originalSend = jest.fn((chunk: unknown) => { + response.end(chunk) + return response + }) + const originalJson = jest.fn((body: unknown) => { + response.send(JSON.stringify(body)) + return response + }) + response = { + status: jest.fn(() => response), + json: originalJson, + send: originalSend, + end: originalEnd + } + const next = jest.fn() + responseSizeLimit('TEST', 256)({} as any, response, next) + + if (method === 'send') response.send(value) + else response.end(value, encoding) + + expect(response.status).toHaveBeenCalledWith(413) + expect(originalJson).toHaveBeenCalledWith( + expect.objectContaining({ code: 'ERR_RESPONSE_TOO_LARGE' }) + ) + expect(originalSend).toHaveBeenCalled() + expect(originalEnd).toHaveBeenCalled() + + response.json({ ignored: true }) + response.send('ignored') + response.end('ignored') + expect(response.status).toHaveBeenCalledTimes(1) + }) }) diff --git a/packages/overlays/overlay-express/src/security/edgePolicy.ts b/packages/overlays/overlay-express/src/security/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/packages/overlays/overlay-express/src/security/edgePolicy.ts +++ b/packages/overlays/overlay-express/src/security/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index 2ad0ad3ba..c84c0c1d0 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay", - "version": "2.2.7", + "version": "2.3.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 75503085a..e16092a54 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -136,6 +136,7 @@ export class Engine { * @param {TopicAnchorHeaderResolver} topicAnchorHeaderResolver - Resolves block hashes for BASM anchors. * @param {boolean} basmSyncEnabled - Whether BASM sync should run automatically. * @param {number} unprovenEvictionBlocks - Default block age for opt-in unproven state eviction. + * @param {number} maxLookupResults - Maximum lookup formulas hydrated per request. Use -1 to opt out. */ constructor( public managers: { [key: string]: TopicManager }, @@ -156,8 +157,12 @@ export class Engine { public suppressDefaultSyncAdvertisements = true, public topicAnchorHeaderResolver?: TopicAnchorHeaderResolver, public basmSyncEnabled = false, - public unprovenEvictionBlocks = 144 + public unprovenEvictionBlocks = 144, + public maxLookupResults = 1000 ) { + if (maxLookupResults !== -1 && (!Number.isSafeInteger(maxLookupResults) || maxLookupResults < 1)) { + throw new TypeError('maxLookupResults must be -1 or a positive safe integer') + } // To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'. // For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates. this.syncConfiguration ??= {} @@ -1137,6 +1142,11 @@ export class Engine { if (lookupService === undefined || lookupService === null) throw new Error(`Lookup service not found for provider: ${lookupQuestion.service}`) const lookupResult = await lookupService.lookup(lookupQuestion) + if (this.maxLookupResults !== -1 && lookupResult.length > this.maxLookupResults) { + throw new RangeError( + `Lookup returned ${lookupResult.length} results; maximum is ${this.maxLookupResults}` + ) + } const hydrationContext = this.createUTXOHistoryHydrationContext() await this.preloadOutputsWithBEEF( lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })), diff --git a/packages/overlays/overlay/src/__tests/Engine.test.ts b/packages/overlays/overlay/src/__tests/Engine.test.ts index d62533369..5763ce5b3 100644 --- a/packages/overlays/overlay/src/__tests/Engine.test.ts +++ b/packages/overlays/overlay/src/__tests/Engine.test.ts @@ -125,6 +125,25 @@ describe('BSV Overlay Services Engine', () => { jest.restoreAllMocks() }) + it('rejects oversized lookup formulas before hydrating their outputs', async () => { + mockLookupService.lookup = jest.fn(async () => [ + { txid: exampleTXID, outputIndex: 0, history: 0 }, + { txid: exampleTXID, outputIndex: 0, history: 0 } + ]) + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorageEngine, + mockChainTracker + ) + engine.maxLookupResults = 1 + + await expect(engine.lookup({ service: 'Hello', query: {} })).rejects.toThrow( + 'Lookup returned 2 results; maximum is 1' + ) + expect(mockStorageEngine.findOutput).not.toHaveBeenCalled() + }) + it('engine.syncAdvertisements should return void when invalid hostingURL is provided', async () => { for (const url of invalidHostingUrls) { const engine = new Engine( diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 96aa3397f..a611326c4 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -6,6 +6,15 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox (unreleased) +- Make ChainTracks credential-free by default on mainnet, testnet, and + TerraTestNet through the public Arcade/go-chaintracks v2 HTTP and SSE APIs. + Add explicit STN and Terra Scaling TestNet support, exact per-network genesis + headers, isolated in-memory storage, and remove silent testnet aliases. +- Add prioritized bulk/live source failover, locally validated last-good height + operation, source health reporting, bounded request timeouts, browser-safe SSE + reconnection, network checks, and a globally rate-limited anonymous + WhatsOnChain fallback for mainnet/testnet. WhatsOnChain keys remain optional + and rejected configured keys retry header/info requests anonymously. - Preserve the default automatically negotiated in-memory `noSend` batching and `sendWith` lifecycle introduced in #289. Expand inherited txid-only proof ancestors for cold clients, preserve caller-declared known txids, report the @@ -170,10 +179,10 @@ attention to changes that materially alter behavior or extend functionality. `auth-express-middleware@2.1.1` likewise declares its `mime-types` runtime import so strict package managers do not fail when loading the built packages. - - Release prep for `2.4.2`: proof completion now discovers every local - transaction row sharing the proven txid, repairs notification-set drift from - concurrent multi-user `internalizeAction` calls, and idempotently completes - any local copy omitted by a last-writer-wins notification update. +- Release prep for `2.4.2`: proof completion now discovers every local + transaction row sharing the proven txid, repairs notification-set drift from + concurrent multi-user `internalizeAction` calls, and idempotently completes + any local copy omitted by a last-writer-wins notification update. - Release prep for `2.4.1`: define one managed-change policy across Knex and IndexedDB allocation, counting, default balance reporting, `balanceAndUtxos`, and `noSendChange`. @@ -216,7 +225,8 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox 2.1.20 - Update cdn.projectbabbage.com valid blockheaders file hash. -- +- + ## wallet-toolbox 2.1.19 - Merge PR#146. GenerateChange change to better handle dust situations. Redundant trimInputBeef knownTxids safety check. @@ -239,7 +249,6 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox 2.1.15 - audit fix - ## wallet-toolbox 2.1.14 @@ -277,7 +286,7 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox 2.1.9 -- Fix batch sending bug in TaskSendWaiting +- Fix batch sending bug in TaskSendWaiting ## wallet-toolbox 2.1.8 diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 8cc54d9a9..d8587372b 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -34,6 +34,34 @@ The toolbox publishes three npm packages from this repo: - **[`@bsv/wallet-toolbox-client`](https://www.npmjs.com/package/@bsv/wallet-toolbox-client)** — Browser build; excludes Node-only backends (Knex/SQLite/MySQL) - **[`@bsv/wallet-toolbox-mobile`](https://www.npmjs.com/package/@bsv/wallet-toolbox-mobile)** — Mobile build; IndexedDB and remote storage only +### ChainTracks sources and networks + +Wallet services do not require a WhatsOnChain key for ChainTracks. Mainnet, +testnet, and TerraTestNet use the public Arcade/go-chaintracks v2 HTTP and SSE +surfaces by default. Bulk batches still pass through local serialization, hash, +continuity, and genesis checks; providers are tried in priority order; and a +synchronized tracker can continue serving its last-good checked data during a +provider outage. WhatsOnChain remains a mainnet/testnet fallback and anonymous +requests are serialized below its documented public rate. + +The supported chain identifiers are `main`, `test`, `stn`, `ttn`, and `tstn` +(`mock` remains available for test utilities). STN and Terra Scaling TestNet do +not have operator-independent public endpoints: set `STN_CHAINTRACKS_URL` or +`TSTN_CHAINTRACKS_URL`, use the matching Arcade environment variable, or inject +an explicit `ChaintracksClientApi`. URLs ending in `/v2` use the reconnecting +go-chaintracks client; existing legacy v1 URLs and explicit clients remain +compatible. Browser and mobile distributions expose the same fetch/SSE client +without Node `Buffer` or filesystem dependencies. + +Arcade is the browser-safe HTTPS/SSE gateway for Teranode-backed header data. +Direct Teranode P2P is not included in browser/mobile artifacts. + +Core ChainTracks factories accept a final source-options argument when an +application must override the defaults. Set `disableChaintracks`, `disableCdn`, +or `disableWhatsOnChain` to `true` to opt out of an automatic source, or pass an +explicit `chaintracks` client to retain an existing deployment topology. All +earlier positional arguments remain unchanged. + ## Getting Started ### Installation diff --git a/packages/wallet/wallet-toolbox/client/package.json b/packages/wallet/wallet-toolbox/client/package.json index d33ff4c6c..13f9fc427 100644 --- a/packages/wallet/wallet-toolbox/client/package.json +++ b/packages/wallet/wallet-toolbox/client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-client", - "version": "2.5.0", + "version": "2.6.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 28454b298..5a71e5404 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,14 +2,14 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1527000, - "gzip": 360000, - "brotli": 283000 + "raw": 1540000, + "gzip": 365000, + "brotli": 286000 }, "esbuild": { - "raw": 1192000, + "raw": 1205000, "gzip": 330000, - "brotli": 265000 + "brotli": 268000 } } } diff --git a/packages/wallet/wallet-toolbox/mobile/package.json b/packages/wallet/wallet-toolbox/mobile/package.json index af43a3b0b..08d414c84 100644 --- a/packages/wallet/wallet-toolbox/mobile/package.json +++ b/packages/wallet/wallet-toolbox/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-mobile", - "version": "2.5.0", + "version": "2.6.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/mobile/vitest.config.ts b/packages/wallet/wallet-toolbox/mobile/vitest.config.ts index 464a6bbb0..09f26905f 100644 --- a/packages/wallet/wallet-toolbox/mobile/vitest.config.ts +++ b/packages/wallet/wallet-toolbox/mobile/vitest.config.ts @@ -7,7 +7,12 @@ export default defineConfig({ include: ['mobile/test/**/*.test.ts'], coverage: { provider: 'v8', - include: ['src/index.mobile.ts', 'src/services/chaintracker/chaintracks/Api/BlockHeaderApi.ts'], + include: [ + 'src/index.mobile.ts', + 'src/services/chaintracker/chaintracks/Api/BlockHeaderApi.ts', + 'src/services/chaintracker/chaintracks/Api/BulkIngestorApi.ts', + 'src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts' + ], reportsDirectory: 'mobile/coverage', reporter: ['text', 'lcov'], thresholds: { diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index 11cba5bd1..c12d0f787 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox", - "version": "2.5.0", + "version": "2.6.0", "sideEffects": false, "type": "commonjs", "engines": { diff --git a/packages/wallet/wallet-toolbox/src/index.all.ts b/packages/wallet/wallet-toolbox/src/index.all.ts index bbc4e51be..960a8b6cd 100644 --- a/packages/wallet/wallet-toolbox/src/index.all.ts +++ b/packages/wallet/wallet-toolbox/src/index.all.ts @@ -13,6 +13,7 @@ export * from './SetupWallet' export * from './CWIStyleWalletManager' export * from './sdk/PrivilegedKeyManager' export * from './services/Services' +export * from './services/createDefaultWalletServicesOptions' export * from './services/providers/ArcSSEClient' export * from './signer/WalletSigner' export * from './SimpleWalletManager' diff --git a/packages/wallet/wallet-toolbox/src/index.mobile.ts b/packages/wallet/wallet-toolbox/src/index.mobile.ts index 403a5df85..1a3a07ed5 100644 --- a/packages/wallet/wallet-toolbox/src/index.mobile.ts +++ b/packages/wallet/wallet-toolbox/src/index.mobile.ts @@ -8,6 +8,7 @@ export * from './CWIStyleWalletManager' export * from './monitor/Monitor' export * from './sdk/PrivilegedKeyManager' export * from './services/Services' +export * from './services/createDefaultWalletServicesOptions' export * from './services/providers/ArcSSEClient' export * from './signer/WalletSigner' export * from './SimpleWalletManager' diff --git a/packages/wallet/wallet-toolbox/src/sdk/types.ts b/packages/wallet/wallet-toolbox/src/sdk/types.ts index f525d6650..c3b057df0 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/types.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/types.ts @@ -14,7 +14,7 @@ export interface OutPoint { vout: number } -export type Chain = 'main' | 'test' | 'ttn' | 'tstn' | 'mock' +export type Chain = 'main' | 'test' | 'stn' | 'ttn' | 'tstn' | 'mock' /** * Initial status (attempts === 0): @@ -81,15 +81,7 @@ export const ProvenTxReqNonTerminalStatus: ProvenTxReqStatus[] = [ ] export type TransactionStatus = - | 'completed' - | 'failed' - | 'unprocessed' - | 'sending' - | 'unproven' - | 'unsigned' - | 'nosend' - | 'nonfinal' - | 'unfail' + 'completed' | 'failed' | 'unprocessed' | 'sending' | 'unproven' | 'unsigned' | 'nosend' | 'nonfinal' | 'unfail' export interface Paged { limit: number @@ -124,7 +116,7 @@ export interface ScriptTemplateUnlock { export interface WalletBalance { total: number - utxos: Array<{ satoshis: number, outpoint: string }> + utxos: Array<{ satoshis: number; outpoint: string }> } export interface ReqHistoryNote { @@ -189,13 +181,10 @@ export const specOpSetWalletChangeParams = 'a4979d28ced8581e9c1c92f1001cc7cb3aab * @param basket Output basket name value. * @returns true iff the `basket` name is a reserved `listOutputs` special operation identifier. */ -export function isListOutputsSpecOp (basket: string): boolean { - return [ - specOpWalletBalance, - specOpWalletManagedUtxos, - specOpInvalidChange, - specOpSetWalletChangeParams - ].includes(basket) +export function isListOutputsSpecOp(basket: string): boolean { + return [specOpWalletBalance, specOpWalletManagedUtxos, specOpInvalidChange, specOpSetWalletChangeParams].includes( + basket + ) } /** @@ -220,7 +209,7 @@ export const specOpFailedActions = '97d4eb1e49215e3374cc2c1939a7c43a55e95c7427bf * @param label Action / Transaction label name value. * @returns true iff the `label` name is a reserved `listActions` special operation identifier. */ -export function isListActionsSpecOp (label: string): boolean { +export function isListActionsSpecOp(label: string): boolean { return [specOpNoSendActions, specOpFailedActions].includes(label) } @@ -236,6 +225,6 @@ export const specOpThrowReviewActions = 'a496e747fc3ad5fabdd4ae8f91184e71f87539b * @param label Action / Transaction label name value. * @returns true iff the `label` name is a reserved `createAction` special operation identifier. */ -export function isCreateActionSpecOp (label: string): boolean { +export function isCreateActionSpecOp(label: string): boolean { return [specOpThrowReviewActions].includes(label) } diff --git a/packages/wallet/wallet-toolbox/src/services/Services.ts b/packages/wallet/wallet-toolbox/src/services/Services.ts index aa54087e1..a02a412ea 100644 --- a/packages/wallet/wallet-toolbox/src/services/Services.ts +++ b/packages/wallet/wallet-toolbox/src/services/Services.ts @@ -76,7 +76,10 @@ export class Services implements WalletServices { this.chain = typeof optionsOrChain === 'string' ? optionsOrChain : optionsOrChain.chain if (this.chain === 'mock') { - throw new WERR_INVALID_PARAMETER('chain', "'main', 'test', 'ttn', or 'tstn'. Use MockServices for 'mock' chain.") + throw new WERR_INVALID_PARAMETER( + 'chain', + "'main', 'test', 'stn', 'ttn', or 'tstn'. Use MockServices for 'mock' chain." + ) } this.options = typeof optionsOrChain === 'string' ? Services.createDefaultOptions(this.chain) : optionsOrChain @@ -101,10 +104,10 @@ export class Services implements WalletServices { const hasBitails = this.chain === 'main' || this.chain === 'test' - // tstn runs only Arcade + ChainTracks; it has no WhatsOnChain / - // block-explorer provider. WhatsOnChain-only lookups therefore remain - // unavailable on tstn by design. - const hasWhatsOnChain = this.chain !== 'tstn' + // The public WhatsOnChain API documents mainnet and testnet only. + // Teranode-family networks use Arcade/ChainTracks and explicit operator + // endpoints instead of being silently aliased to testnet. + const hasWhatsOnChain = this.chain === 'main' || this.chain === 'test' if (hasBitails) { this.bitails = new Bitails(this.chain, { apiKey: this.options.bitailsApiKey }) diff --git a/packages/wallet/wallet-toolbox/src/services/__tests/Services.arcade.test.ts b/packages/wallet/wallet-toolbox/src/services/__tests/Services.arcade.test.ts index 26263e61a..494f7cc4f 100644 --- a/packages/wallet/wallet-toolbox/src/services/__tests/Services.arcade.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/__tests/Services.arcade.test.ts @@ -10,10 +10,10 @@ import { arcadeDefaultUrl, createDefaultWalletServicesOptions } from '../createD const ARCADE_URL = 'https://arcade-v2-ttn-us-1.bsvblockchain.tech' describe('Services Arcade wiring', () => { - test('arcadeDefaultUrl maps known chains and omits testnet', () => { + test('arcadeDefaultUrl maps all public Arcade deployments', () => { expect(arcadeDefaultUrl('main')).toBe('https://arcade-v2-us-1.bsvblockchain.tech') expect(arcadeDefaultUrl('ttn')).toBe('https://arcade-v2-ttn-us-1.bsvblockchain.tech') - expect(arcadeDefaultUrl('test')).toBeUndefined() + expect(arcadeDefaultUrl('test')).toBe('https://arcade-v2-testnet-us-1.bsvblockchain.tech') }) test('arcadeDefaultUrl for tstn is driven by TSTN_ARCADE_URL', () => { @@ -70,7 +70,14 @@ describe('Services Arcade wiring', () => { test('explicit empty-string arcadeUrl keeps Arcade disabled', () => { const options = createDefaultWalletServicesOptions( - 'test', undefined, undefined, undefined, undefined, undefined, undefined, undefined, + 'test', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, '' // arcadeUrl explicitly empty ) expect(options.arcadeUrl).toBeUndefined() diff --git a/packages/wallet/wallet-toolbox/src/services/__tests/Services.chaintracksNetworks.test.ts b/packages/wallet/wallet-toolbox/src/services/__tests/Services.chaintracksNetworks.test.ts new file mode 100644 index 000000000..900e973aa --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/services/__tests/Services.chaintracksNetworks.test.ts @@ -0,0 +1,60 @@ +import { Services } from '../Services' +import { ChaintracksServiceClient } from '../chaintracker/chaintracks/ChaintracksServiceClient' +import { GoChaintracksServiceClient } from '../chaintracker/chaintracks/GoChaintracksServiceClient' +import { + arcadeDefaultUrl, + arcDefaultUrl, + createDefaultWalletServicesOptions +} from '../createDefaultWalletServicesOptions' + +describe('ChainTracks network defaults', () => { + let stnChaintracks: string | undefined + let stnArcade: string | undefined + + beforeEach(() => { + stnChaintracks = process.env.STN_CHAINTRACKS_URL + stnArcade = process.env.STN_ARCADE_URL + process.env.STN_CHAINTRACKS_URL = 'https://stn.example/chaintracks/v2' + process.env.STN_ARCADE_URL = 'https://stn.example' + }) + + afterEach(() => { + if (stnChaintracks == null) delete process.env.STN_CHAINTRACKS_URL + else process.env.STN_CHAINTRACKS_URL = stnChaintracks + if (stnArcade == null) delete process.env.STN_ARCADE_URL + else process.env.STN_ARCADE_URL = stnArcade + }) + + test.each(['main', 'test', 'ttn'] as const)('%s defaults to the credential-free v2 client', chain => { + expect(createDefaultWalletServicesOptions(chain).chaintracks).toBeInstanceOf(GoChaintracksServiceClient) + }) + + test('stn uses an explicit v2 endpoint without adding explorer fallbacks', () => { + const options = createDefaultWalletServicesOptions('stn') + expect(options.chaintracks).toBeInstanceOf(GoChaintracksServiceClient) + const services = new Services(options) + const names = [ + ...services.getMerklePathServices.services, + ...services.getRawTxServices.services, + ...services.getUtxoStatusServices.services, + ...services.getStatusForTxidsServices.services, + ...services.getScriptHashHistoryServices.services + ].map(service => service.name) + expect(names).not.toContain('WhatsOnChain') + expect(names).not.toContain('Bitails') + }) + + test('stn falls back to an operator Arcade v1 path without aliasing another network', () => { + delete process.env.STN_CHAINTRACKS_URL + process.env.STN_ARCADE_URL = 'https://stn.example///' + + expect(arcadeDefaultUrl('stn')).toBe('https://stn.example///') + expect(arcDefaultUrl('stn')).toBe('https://stn.example///') + expect(createDefaultWalletServicesOptions('stn').chaintracks).toBeInstanceOf(ChaintracksServiceClient) + }) + + test('default service factories reject mock-chain construction', () => { + expect(() => createDefaultWalletServicesOptions('mock')).toThrow("does not support 'mock' chain") + expect(() => new Services('mock')).toThrow("Use MockServices for 'mock' chain") + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/BulkIngestorApi.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/BulkIngestorApi.ts index 7fde28901..d6257d94f 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/BulkIngestorApi.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/BulkIngestorApi.ts @@ -5,7 +5,7 @@ import { ChaintracksStorageApi } from './ChaintracksStorageApi' export interface BulkIngestorBaseOptions { /** - * The target chain: "main" or "test" + * The target chain. */ chain: Chain diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts index 631c295d4..75504b514 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts @@ -36,6 +36,18 @@ export interface ChaintracksInfoApi { bulkIngestors: string[] liveIngestors: string[] packages: ChaintracksPackageInfoApi[] + /** Last observed source state. Additive and omitted by older services. */ + sources?: ChaintracksSourceStatusApi[] +} + +/** @public */ +export interface ChaintracksSourceStatusApi { + name: string + role: 'bulk' | 'live' + state: 'unknown' | 'healthy' | 'degraded' + lastSuccess?: string + lastFailure?: string + error?: string } /** diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Chaintracks.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Chaintracks.ts index f96a27e77..db44012af 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Chaintracks.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Chaintracks.ts @@ -7,7 +7,12 @@ import { validateAgainstDirtyHashes } from './util/dirtyHashes' import { ChaintracksOptions, ChaintracksManagementApi } from './Api/ChaintracksApi' import { blockHash, validateHeaderFormat } from './util/blockHeaderUtilities' import { Chain } from '../../../sdk/types' -import { ChaintracksInfoApi, HeaderListener, ReorgListener } from './Api/ChaintracksClientApi' +import { + ChaintracksInfoApi, + ChaintracksSourceStatusApi, + HeaderListener, + ReorgListener +} from './Api/ChaintracksClientApi' import { BaseBlockHeader, BlockHeader, LiveBlockHeader } from './Api/BlockHeaderApi' import { asString } from '../../../utility/utilityHelpers.noBuffer' import { HeightRange, HeightRanges } from './util/HeightRange' @@ -16,7 +21,7 @@ import { ChaintracksFsApi } from './Api/ChaintracksFsApi' import { randomBytesBase64, wait } from '../../../utility/utilityHelpers' import { WalletError } from '../../../sdk/WalletError' export class Chaintracks implements ChaintracksManagementApi { - static createOptions (chain: Chain): ChaintracksOptions { + static createOptions(chain: Chain): ChaintracksOptions { return { chain, storage: undefined, @@ -37,7 +42,10 @@ export class Chaintracks implements ChaintracksManagementApi { // Collection of all long running "threads": main thread (liveHeaders consumer / monitor) and each live header ingestor. private readonly promises: Array> = [] - private readonly callbacks: { header: Record, reorg: Record } = { header: {}, reorg: {} } + private readonly callbacks: { + header: Record + reorg: Record + } = { header: {}, reorg: {} } private readonly storage: ChaintracksStorageApi private readonly bulkIngestors: BulkIngestorApi[] private readonly liveIngestors: LiveIngestorApi[] @@ -52,21 +60,34 @@ export class Chaintracks implements ChaintracksManagementApi { private subscriberCallbacksEnabled = false private stopMainThread = true - private lastPresentHeight = 0 + private lastPresentHeight = -1 private lastPresentHeightMsecs = 0 private readonly lastPresentHeightMaxAge = 60 * 1000 // 1 minute, in milliseconds private readonly lock = new SingleWriterMultiReaderLock() + private readonly sourceStatus = new Map() - constructor (public options: ChaintracksOptions) { + constructor(public options: ChaintracksOptions) { if (options.storage == null) throw new Error('storage is required.') - if (!options.bulkIngestors || options.bulkIngestors.length < 1) { throw new Error('At least one bulk ingestor is required.') } - if (!options.liveIngestors || options.liveIngestors.length < 1) { throw new Error('At least one live ingestor is required.') } + if (!options.bulkIngestors || options.bulkIngestors.length < 1) { + throw new Error('At least one bulk ingestor is required.') + } + if (!options.liveIngestors || options.liveIngestors.length < 1) { + throw new Error('At least one live ingestor is required.') + } this.chain = options.chain this.readonly = options.readonly this.storage = options.storage this.bulkIngestors = options.bulkIngestors this.liveIngestors = options.liveIngestors + for (const [index, source] of this.bulkIngestors.entries()) { + const name = this.sourceName('bulk', index, source) + this.sourceStatus.set(name, { name, role: 'bulk', state: 'unknown' }) + } + for (const [index, source] of this.liveIngestors.entries()) { + const name = this.sourceName('live', index, source) + this.sourceStatus.set(name, { name, role: 'live', state: 'unknown' }) + } this.addLiveRecursionLimit = options.addLiveRecursionLimit @@ -76,7 +97,7 @@ export class Chaintracks implements ChaintracksManagementApi { this.log(`New ChaintracksBase Instance Constructed ${options.chain}Net`) } - async getChain (): Promise { + async getChain(): Promise { return this.chain } @@ -84,44 +105,61 @@ export class Chaintracks implements ChaintracksManagementApi { * Caches and returns most recently sourced value if less than one minute old. * @returns the current externally available chain height (via bulk ingestors). */ - async getPresentHeight (): Promise { + async getPresentHeight(): Promise { const now = Date.now() - if (this.lastPresentHeight && now - this.lastPresentHeightMsecs < this.lastPresentHeightMaxAge) { + if (this.lastPresentHeight >= 0 && now - this.lastPresentHeightMsecs < this.lastPresentHeightMaxAge) { return this.lastPresentHeight } - const presentHeights: number[] = [] - for (const bulk of this.bulkIngestors) { + for (const [index, bulk] of this.bulkIngestors.entries()) { + const source = this.sourceName('bulk', index, bulk) try { const presentHeight = await bulk.getPresentHeight() - if (presentHeight) presentHeights.push(presentHeight) + if (presentHeight != null && Number.isInteger(presentHeight) && presentHeight >= 0) { + this.markSourceSuccess(source, 'bulk') + this.lastPresentHeight = presentHeight + this.lastPresentHeightMsecs = now + return presentHeight + } } catch (uerr: unknown) { - console.error(uerr) + const error = WalletError.fromUnknown(uerr) + this.markSourceFailure(source, 'bulk', error) + this.log(`Present-height source ${source} failed: ${error.message}`) + } + } + + // A provider outage must not make an already synchronized tracker unusable. + if (this.lastPresentHeight >= 0) return this.lastPresentHeight + try { + const ranges = await this.storage.getAvailableHeightRanges() + const localHeight = Math.max(ranges.bulk.maxHeight, ranges.live.maxHeight) + if (localHeight >= 0) { + this.lastPresentHeight = localHeight + this.lastPresentHeightMsecs = now + return localHeight } + } catch (error: unknown) { + this.log(`Unable to read the locally validated ChainTracks height: ${WalletError.fromUnknown(error).message}`) } - const presentHeight = (presentHeights.length > 0) ? Math.max(...presentHeights) : undefined - if (!presentHeight) throw new Error('At least one bulk ingestor must implement getPresentHeight.') - this.lastPresentHeight = presentHeight - this.lastPresentHeightMsecs = now - return presentHeight + throw new Error('No present-height source or locally validated headers are available.') } - async currentHeight (): Promise { + async currentHeight(): Promise { return await this.getPresentHeight() } - async subscribeHeaders (listener: HeaderListener): Promise { + async subscribeHeaders(listener: HeaderListener): Promise { const ID = randomBytesBase64(8) this.callbacks.header[ID] = listener return ID } - async subscribeReorgs (listener: ReorgListener): Promise { + async subscribeReorgs(listener: ReorgListener): Promise { const ID = randomBytesBase64(8) this.callbacks.reorg[ID] = listener return ID } - async unsubscribe (subscriptionId: string): Promise { + async unsubscribe(subscriptionId: string): Promise { let success = true if (this.callbacks.header[subscriptionId]) this.callbacks.header[subscriptionId] = null else if (this.callbacks.reorg[subscriptionId]) this.callbacks.reorg[subscriptionId] = null @@ -138,7 +176,7 @@ export class Chaintracks implements ChaintracksManagementApi { * * @param header */ - async addHeader (header: BaseBlockHeader): Promise { + async addHeader(header: BaseBlockHeader): Promise { this.baseHeaders.push(header) } @@ -151,7 +189,7 @@ export class Chaintracks implements ChaintracksManagementApi { * * @returns when available for client requests */ - async makeAvailable (): Promise { + async makeAvailable(): Promise { if (this.available) return await this.lock.withWriteLock(async () => { // Only the first call proceeds to initialize... @@ -163,13 +201,15 @@ export class Chaintracks implements ChaintracksManagementApi { // Start all live ingestors to push new headers onto liveHeaders... each long running. this.stopMainThread = false - for (const liveIngestor of this.liveIngestors) this.promises.push(this.runLiveIngestor(liveIngestor)) + for (const [index, liveIngestor] of this.liveIngestors.entries()) { + this.promises.push(this.runLiveIngestor(liveIngestor, index)) + } // Start mai loop to shift out liveHeaders...once sync'd, will set `available` true. this.promises.push(this.mainThreadShiftLiveHeaders()) // Wait for the main thread to finish initial sync. - while (!this.available && (this.startupError == null)) { + while (!this.available && this.startupError == null) { await wait(100) } @@ -177,11 +217,11 @@ export class Chaintracks implements ChaintracksManagementApi { }) } - async startPromises (): Promise { + async startPromises(): Promise { if (this.promises.length > 0 || !this.stopMainThread) return } - async destroy (): Promise { + async destroy(): Promise { if (!this.available) return await this.lock.withWriteLock(async () => { if (!this.available || this.stopMainThread) return @@ -197,16 +237,18 @@ export class Chaintracks implements ChaintracksManagementApi { }) } - async listening (): Promise { + async listening(): Promise { return await this.makeAvailable() } - private async runLiveIngestor (liveIngestor: LiveIngestorApi): Promise { + private async runLiveIngestor(liveIngestor: LiveIngestorApi, index: number): Promise { let restartCount = 0 const name = liveIngestor.constructor.name + const source = this.sourceName('live', index, liveIngestor) while (!this.stopMainThread) { try { + this.markSourceSuccess(source, 'live') await liveIngestor.startListening(this.liveHeaders) if (this.stopMainThread) return restartCount++ @@ -217,57 +259,60 @@ export class Chaintracks implements ChaintracksManagementApi { if (this.stopMainThread) return restartCount++ const e = WalletError.fromUnknown(error_) + this.markSourceFailure(source, 'live', e) const waitMsecs = this.liveIngestorRestartWaitMsecs(restartCount) - this.log(`Live ingestor ${name} failed restart=${restartCount} retryMsecs=${waitMsecs}: ${e.stack ?? e.message}`) + this.log( + `Live ingestor ${name} failed restart=${restartCount} retryMsecs=${waitMsecs}: ${e.stack ?? e.message}` + ) await wait(waitMsecs) } } } - private liveIngestorRestartWaitMsecs (restartCount: number): number { + private liveIngestorRestartWaitMsecs(restartCount: number): number { return Math.min(1000 * Math.min(2 ** Math.max(restartCount - 1, 0), 60), 60000) } - async isListening (): Promise { + async isListening(): Promise { return this.available } - async isSynchronized (): Promise { + async isSynchronized(): Promise { await this.makeAvailable() return true } - async findHeaderForHeight (height: number): Promise { + async findHeaderForHeight(height: number): Promise { await this.makeAvailable() return await this.lock.withReadLock(async () => await this.findHeaderForHeightNoLock(height)) } - private async findHeaderForHeightNoLock (height: number): Promise { + private async findHeaderForHeightNoLock(height: number): Promise { return await this.storage.findHeaderForHeightOrUndefined(height) } - async findHeaderForBlockHash (hash: string): Promise { + async findHeaderForBlockHash(hash: string): Promise { await this.makeAvailable() return await this.lock.withReadLock(async () => await this.findHeaderForBlockHashNoLock(hash)) } - private async findHeaderForBlockHashNoLock (hash: string): Promise { + private async findHeaderForBlockHashNoLock(hash: string): Promise { return (await this.storage.findLiveHeaderForBlockHash(hash)) || undefined } - async isValidRootForHeight (root: string, height: number): Promise { + async isValidRootForHeight(root: string, height: number): Promise { const r = await this.findHeaderForHeight(height) if (r == null) return false const isValid = root === r.merkleRoot return isValid } - async getInfo (): Promise { + async getInfo(): Promise { await this.makeAvailable() return await this.lock.withReadLock(async () => await this.getInfoNoLock()) } - private async getInfoNoLock (): Promise { + private async getInfoNoLock(): Promise { const liveRange = await this.storage.findLiveHeightRange() const info: ChaintracksInfoApi = { chain: this.chain, @@ -276,33 +321,34 @@ export class Chaintracks implements ChaintracksManagementApi { storage: this.storage.constructor.name, bulkIngestors: this.bulkIngestors.map(bulkIngestor => bulkIngestor.constructor.name), liveIngestors: this.liveIngestors.map(liveIngestor => liveIngestor.constructor.name), - packages: [] + packages: [], + sources: Array.from(this.sourceStatus.values()).map(status => ({ ...status })) } return info } - async getHeaders (height: number, count: number): Promise { + async getHeaders(height: number, count: number): Promise { await this.makeAvailable() return await this.lock.withReadLock(async () => asString(await this.storage.getHeadersUint8Array(height, count))) } - async findChainTipHeader (): Promise { + async findChainTipHeader(): Promise { await this.makeAvailable() return await this.lock.withReadLock(async () => await this.storage.findChainTipHeader()) } - async findChainTipHash (): Promise { + async findChainTipHash(): Promise { await this.makeAvailable() return await this.lock.withReadLock(async () => await this.storage.findChainTipHash()) } - async findLiveHeaderForBlockHash (hash: string): Promise { + async findLiveHeaderForBlockHash(hash: string): Promise { await this.makeAvailable() const header = await this.lock.withReadLock(async () => await this.storage.findLiveHeaderForBlockHash(hash)) return header || undefined } - async findChainWorkForBlockHash (hash: string): Promise { + async findChainWorkForBlockHash(hash: string): Promise { const header = await this.findLiveHeaderForBlockHash(hash) return header?.chainWork } @@ -310,7 +356,7 @@ export class Chaintracks implements ChaintracksManagementApi { /** * @returns true iff all headers from height zero through current chainTipHeader height can be retreived and form a valid chain. */ - async validate (): Promise { + async validate(): Promise { let h = await this.findChainTipHeader() while (h.height > 0) { const hp = await this.findHeaderForHeight(h.height - 1) @@ -322,7 +368,7 @@ export class Chaintracks implements ChaintracksManagementApi { return true } - async exportBulkHeaders ( + async exportBulkHeaders( toFolder: string, toFs: ChaintracksFsApi, sourceUrl?: string, @@ -334,15 +380,15 @@ export class Chaintracks implements ChaintracksManagementApi { await bulk.exportHeadersToFs(toFs, toHeadersPerFile, toFolder, sourceUrl, maxHeight) } - async startListening (): Promise { + async startListening(): Promise { this.makeAvailable() } - private async syncBulkStorage (presentHeight: number, initialRanges: HeightRanges): Promise { + private async syncBulkStorage(presentHeight: number, initialRanges: HeightRanges): Promise { await this.lock.withWriteLock(async () => await this.syncBulkStorageNoLock(presentHeight, initialRanges)) } - private async syncBulkStorageNoLock (presentHeight: number, initialRanges: HeightRanges): Promise { + private async syncBulkStorageNoLock(presentHeight: number, initialRanges: HeightRanges): Promise { let newLiveHeaders: BlockHeader[] = [] let before = initialRanges let after = before @@ -359,11 +405,15 @@ export class Chaintracks implements ChaintracksManagementApi { if (this.startupError != null) break if (result.done) break if (!result.madeProgress) { - this.log(`Bulk sync stalled after round ${round}. Deferring further bulk sync attempts to continue live header processing.`) + this.log( + `Bulk sync stalled after round ${round}. Deferring further bulk sync attempts to continue live header processing.` + ) break } if (round === maxSyncRounds) { - this.log(`Bulk sync paused after ${maxSyncRounds} rounds to avoid runaway retries. Will retry in a later sync cycle.`) + this.log( + `Bulk sync paused after ${maxSyncRounds} rounds to avoid runaway retries. Will retry in a later sync cycle.` + ) } } @@ -379,57 +429,98 @@ export class Chaintracks implements ChaintracksManagementApi { } } - private async runBulkSyncRound ( + private async runBulkSyncRound( before: HeightRanges, presentHeight: number, newLiveHeaders: BlockHeader[] - ): Promise<{ after: HeightRanges, newLiveHeaders: BlockHeader[], done: boolean, madeProgress: boolean }> { + ): Promise<{ after: HeightRanges; newLiveHeaders: BlockHeader[]; done: boolean; madeProgress: boolean }> { let after = before let bulkSyncError: WalletError | undefined let madeProgress = false let hadSuccess = false let done = false - for (const bulk of this.bulkIngestors) { + for (const [index, bulk] of this.bulkIngestors.entries()) { + const source = this.sourceName('bulk', index, bulk) try { const beforeBulkMax = before.bulk.maxHeight const beforeLiveRange = HeightRange.from(newLiveHeaders) const r = await bulk.synchronize(presentHeight, before, newLiveHeaders) hadSuccess = true + this.markSourceSuccess(source, 'bulk') newLiveHeaders = r.liveHeaders after = await this.storage.getAvailableHeightRanges() const added = after.bulk.above(before.bulk) const afterLiveRange = HeightRange.from(newLiveHeaders) - if (after.bulk.maxHeight > beforeBulkMax || afterLiveRange.maxHeight > beforeLiveRange.maxHeight) madeProgress = true + if (after.bulk.maxHeight > beforeBulkMax || afterLiveRange.maxHeight > beforeLiveRange.maxHeight) + madeProgress = true before = after - this.log(`Bulk Ingestor: ${added.length} added with ${newLiveHeaders.length} live headers from ${bulk.constructor.name}`) - if (r.done) { done = true; break } + this.log( + `Bulk Ingestor: ${added.length} added with ${newLiveHeaders.length} live headers from ${bulk.constructor.name}` + ) + if (r.done) { + done = true + break + } } catch (error_: unknown) { const e = (bulkSyncError = WalletError.fromUnknown(error_)) + this.markSourceFailure(source, 'bulk', e) this.log(`bulk sync error: ${e.message}`) - // During initial startup, bulk ingestors must be available. - if (!this.available) break } } - if (!this.available && (bulkSyncError != null) && !hadSuccess) this.startupError = bulkSyncError + if (!this.available && bulkSyncError != null && !hadSuccess) this.startupError = bulkSyncError return { after, newLiveHeaders, done, madeProgress } } - private async getMissingBlockHeader (hash: string): Promise { - for (const live of this.liveIngestors) { - const header = await live.getHeaderByHash(hash) - if (header != null) return header + private sourceName(role: 'bulk' | 'live', index: number, source: object): string { + return `${role}[${index}]:${source.constructor.name}` + } + + private markSourceSuccess(name: string, role: 'bulk' | 'live'): void { + this.sourceStatus.set(name, { + ...this.sourceStatus.get(name), + name, + role, + state: 'healthy', + lastSuccess: new Date().toISOString(), + error: undefined + }) + } + + private markSourceFailure(name: string, role: 'bulk' | 'live', error: Error): void { + this.sourceStatus.set(name, { + ...this.sourceStatus.get(name), + name, + role, + state: 'degraded', + lastFailure: new Date().toISOString(), + error: error.message + }) + } + + private async getMissingBlockHeader(hash: string): Promise { + for (const [index, live] of this.liveIngestors.entries()) { + const source = this.sourceName('live', index, live) + try { + const header = await live.getHeaderByHash(hash) + this.markSourceSuccess(source, 'live') + if (header != null) return header + } catch (error: unknown) { + const resolved = WalletError.fromUnknown(error) + this.markSourceFailure(source, 'live', resolved) + this.log(`Header lookup source ${source} failed: ${resolved.message}`) + } } return undefined } - private invalidInsertHeaderResult (ihr: InsertHeaderResult): boolean { + private invalidInsertHeaderResult(ihr: InsertHeaderResult): boolean { return ihr.noActiveAncestor || ihr.noTip || ihr.badPrev } - private async addLiveHeader (header: BlockHeader): Promise { + private async addLiveHeader(header: BlockHeader): Promise { validateHeaderFormat(header) validateAgainstDirtyHashes(header.hash) @@ -441,7 +532,7 @@ export class Chaintracks implements ChaintracksManagementApi { if (this.subscriberCallbacksEnabled && ihr.added && ihr.isActiveTip) { this.notifyHeaderListeners(header) - if (ihr.reorgDepth > 0 && (ihr.priorTip != null)) { + if (ihr.reorgDepth > 0 && ihr.priorTip != null) { this.notifyReorgListeners(ihr, header) } } @@ -449,22 +540,30 @@ export class Chaintracks implements ChaintracksManagementApi { return ihr } - private notifyHeaderListeners (header: BlockHeader): void { + private notifyHeaderListeners(header: BlockHeader): void { for (const id in this.callbacks.header) { const listener = this.callbacks.header[id] if (listener != null) { - try { listener(header) } catch { /* ignore all errors thrown */ } + try { + listener(header) + } catch { + /* ignore all errors thrown */ + } } } } - private notifyReorgListeners (ihr: InsertHeaderResult, header: BlockHeader): void { + private notifyReorgListeners(ihr: InsertHeaderResult, header: BlockHeader): void { const priorTip: BlockHeader = { ...ihr.priorTip! } const deactivated: BlockHeader[] = ihr.deactivatedHeaders.map(lbh => ({ ...lbh })) for (const id in this.callbacks.reorg) { const listener = this.callbacks.reorg[id] if (listener != null) { - try { listener(ihr.reorgDepth, priorTip, header, deactivated) } catch { /* ignore all errors thrown */ } + try { + listener(ihr.reorgDepth, priorTip, header, deactivated) + } catch { + /* ignore all errors thrown */ + } } } } @@ -481,7 +580,7 @@ export class Chaintracks implements ChaintracksManagementApi { * * Periodically CDN bulk ingestor is invoked to check if incremental headers can be migrated to CDN backed files. */ - private async mainThreadShiftLiveHeaders (): Promise { + private async mainThreadShiftLiveHeaders(): Promise { this.stopMainThread = false let lastSyncCheck = 0 let lastBulkSync = Date.now() @@ -507,7 +606,7 @@ export class Chaintracks implements ChaintracksManagementApi { } /** Returns (potentially updated) lastBulkSync timestamp. */ - private async runBulkSyncIfNeeded (now: number, lastBulkSync: number, cdnSyncRepeatMsecs: number): Promise { + private async runBulkSyncIfNeeded(now: number, lastBulkSync: number, cdnSyncRepeatMsecs: number): Promise { const presentHeight = await this.getPresentHeight() const before = await this.storage.getAvailableHeightRanges() @@ -532,9 +631,7 @@ export class Chaintracks implements ChaintracksManagementApi { return lastBulkSync } - private async processNextQueuedHeader ( - stats: { count: number, liveHeaderDupes: number } - ): Promise { + private async processNextQueuedHeader(stats: { count: number; liveHeaderDupes: number }): Promise { const liveHeader = this.liveHeaders.shift() if (liveHeader != null) { const result = await this.processOneLiveHeader(liveHeader) @@ -549,57 +646,46 @@ export class Chaintracks implements ChaintracksManagementApi { return false } - private async flushLiveHeaderProgress ( - stats: { count: number, liveHeaderDupes: number } - ): Promise { + private async flushLiveHeaderProgress(stats: { count: number; liveHeaderDupes: number }): Promise { if (stats.count === 0) return if (stats.liveHeaderDupes > 0) { this.log(`${stats.liveHeaderDupes} duplicate headers ignored.`) stats.liveHeaderDupes = 0 } const updated = await this.storage.getAvailableHeightRanges() - this.log( - `After adding ${stats.count} live headers\n After live: bulk ${updated.bulk}, live ${updated.live}\n` - ) + this.log(`After adding ${stats.count} live headers\n After live: bulk ${updated.bulk}, live ${updated.live}\n`) stats.count = 0 } - private async waitForQueuedHeaders ( - stats: { count: number, liveHeaderDupes: number }, + private async waitForQueuedHeaders( + stats: { count: number; liveHeaderDupes: number }, lastSyncCheck: number, syncCheckRepeatMsecs: number ): Promise { await this.flushLiveHeaderProgress(stats) await this.checkAndEnableSubscribers() if (!this.available) this.available = true - const needSyncCheck = - Date.now() - lastSyncCheck > syncCheckRepeatMsecs + const needSyncCheck = Date.now() - lastSyncCheck > syncCheckRepeatMsecs if (!needSyncCheck) await wait(1000) return needSyncCheck } - private async processLiveHeaderQueue (lastSyncCheck: number, syncCheckRepeatMsecs: number): Promise { + private async processLiveHeaderQueue(lastSyncCheck: number, syncCheckRepeatMsecs: number): Promise { const stats = { count: 0, liveHeaderDupes: 0 } let needSyncCheck = false while (!needSyncCheck && !this.stopMainThread) { const queuedResult = await this.processNextQueuedHeader(stats) - needSyncCheck = - queuedResult ?? - (await this.waitForQueuedHeaders( - stats, - lastSyncCheck, - syncCheckRepeatMsecs - )) + needSyncCheck = queuedResult ?? (await this.waitForQueuedHeaders(stats, lastSyncCheck, syncCheckRepeatMsecs)) } } - private formatIhrLog (prefix: string, header: BlockHeader, ihr: InsertHeaderResult): string { + private formatIhrLog(prefix: string, header: BlockHeader, ihr: InsertHeaderResult): string { return `${prefix} ${header.height}${ihr.added ? ' added' : ''}${ihr.dupe ? ' dupe' : ''}${ihr.isActiveTip ? ' isActiveTip' : ''}${ihr.reorgDepth ? ' reorg depth ' + ihr.reorgDepth : ''}${ihr.noPrev ? ' noPrev' : ''}${ihr.noActiveAncestor || ihr.noTip || ihr.badPrev ? ' error' : ''}` } - private async processOneLiveHeader ( + private async processOneLiveHeader( startHeader: BlockHeader - ): Promise<{ needSyncCheck: boolean, dupe: boolean, added: boolean }> { + ): Promise<{ needSyncCheck: boolean; dupe: boolean; added: boolean }> { let header = startHeader let recursions = this.addLiveRecursionLimit @@ -612,12 +698,16 @@ export class Chaintracks implements ChaintracksManagementApi { if (ihr.noPrev) { // Previous header is unknown; request it by hash from the network and try adding it first. if (recursions-- <= 0) { - this.log(`Ignoring liveHeader ${header.height} ${header.hash} addLiveRecursionLimit=${this.addLiveRecursionLimit} exceeded.`) + this.log( + `Ignoring liveHeader ${header.height} ${header.hash} addLiveRecursionLimit=${this.addLiveRecursionLimit} exceeded.` + ) return { needSyncCheck: true, dupe: false, added: false } } const prevHeader = await this.getMissingBlockHeader(header.previousHash) if (prevHeader == null) { - this.log(`Ignoring liveHeader ${header.height} ${header.hash} failed to find previous header by hash ${asString(header.previousHash)}`) + this.log( + `Ignoring liveHeader ${header.height} ${header.hash} failed to find previous header by hash ${asString(header.previousHash)}` + ) return { needSyncCheck: true, dupe: false, added: false } } // Retry adding prevHeader first; then re-queue current header. @@ -631,7 +721,7 @@ export class Chaintracks implements ChaintracksManagementApi { return { needSyncCheck: false, dupe: false, added: false } } - private async processOneBaseHeader (bheader: BaseBlockHeader): Promise { + private async processOneBaseHeader(bheader: BaseBlockHeader): Promise { const prev = await this.storage.findLiveHeaderForBlockHash(bheader.previousHash) if (prev == null) { // Unknown previous hash — ignore without triggering a re-sync. @@ -648,7 +738,7 @@ export class Chaintracks implements ChaintracksManagementApi { return ihr.added } - private async checkAndEnableSubscribers (): Promise { + private async checkAndEnableSubscribers(): Promise { if (this.subscriberCallbacksEnabled) return const live = await this.storage.findLiveHeightRange() if (!live.isEmpty) { diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts index b7177b76a..b0b7c8b67 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts @@ -10,6 +10,12 @@ export interface GoChaintracksServiceClientOptions { */ apiPrefix?: string fetch?: typeof fetch + /** Timeout for HTTP requests and the initial SSE handshake. */ + requestTimeoutMsecs?: number + /** Initial delay before reconnecting a closed or failed SSE stream. */ + reconnectWaitMsecs?: number + /** Maximum SSE reconnect delay. */ + reconnectWaitMaxMsecs?: number } interface GoChaintracksHeightResponse { @@ -36,10 +42,13 @@ interface SseSubscription { export class GoChaintracksServiceClient implements ChaintracksClientApi { private readonly baseUrl: string private readonly fetcher: typeof fetch + private readonly requestTimeoutMsecs: number + private readonly reconnectWaitMsecs: number + private readonly reconnectWaitMaxMsecs: number private readonly subscriptions = new Map() private nextSubscriptionId = 1 - constructor ( + constructor( public chain: Chain, serviceUrl: string, options: GoChaintracksServiceClientOptions = {} @@ -53,27 +62,38 @@ export class GoChaintracksServiceClient implements ChaintracksClientApi { } this.baseUrl = `${base}${prefix}` this.fetcher = options.fetch ?? fetch + this.requestTimeoutMsecs = options.requestTimeoutMsecs ?? 30000 + this.reconnectWaitMsecs = options.reconnectWaitMsecs ?? 1000 + this.reconnectWaitMaxMsecs = options.reconnectWaitMaxMsecs ?? 60000 + for (const [name, value] of [ + ['requestTimeoutMsecs', this.requestTimeoutMsecs], + ['reconnectWaitMsecs', this.reconnectWaitMsecs], + ['reconnectWaitMaxMsecs', this.reconnectWaitMaxMsecs] + ] as const) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer.`) + } + } + if (this.reconnectWaitMaxMsecs < this.reconnectWaitMsecs) { + throw new Error('reconnectWaitMaxMsecs must be greater than or equal to reconnectWaitMsecs.') + } } - async currentHeight (): Promise { + async currentHeight(): Promise { return await this.getPresentHeight() } - async isValidRootForHeight (root: string, height: number): Promise { + async isValidRootForHeight(root: string, height: number): Promise { const h = await this.findHeaderForHeight(height) return h != null && root === asString(h.merkleRoot) } - async getChain (): Promise { - try { - const r = await this.getJson('/network') - return this.normalizeChain(r.network) - } catch { - return this.chain - } + async getChain(): Promise { + const r = await this.getJson('/network') + return this.normalizeChain(typeof r === 'string' ? r : r.network) } - async getInfo (): Promise { + async getInfo(): Promise { const tip = await this.findChainTipHeader() return { chain: await this.getChain(), @@ -86,44 +106,45 @@ export class GoChaintracksServiceClient implements ChaintracksClientApi { } } - async getPresentHeight (): Promise { - return (await this.getJson('/height')).height + async getPresentHeight(): Promise { + const result = await this.getJson('/height') + return typeof result === 'number' ? result : result.height } - async getHeaders (height: number, count: number): Promise { + async getHeaders(height: number, count: number): Promise { const bytes = await this.getBinary(`/headers.bin?height=${height}&count=${count}`) - return Buffer.from(bytes).toString('hex') + return asString(bytes) } - async findChainTipHeader (): Promise { + async findChainTipHeader(): Promise { return await this.getJson('/tip') } - async findChainTipHash (): Promise { + async findChainTipHash(): Promise { return (await this.findChainTipHeader()).hash } - async findHeaderForHeight (height: number): Promise { + async findHeaderForHeight(height: number): Promise { return await this.getJsonOrUndefined(`/header/height/${height}`) } - async findHeaderForBlockHash (hash: string): Promise { + async findHeaderForBlockHash(hash: string): Promise { return await this.getJsonOrUndefined(`/header/hash/${hash}`) } - async addHeader (_header: BaseBlockHeader): Promise { + async addHeader(_header: BaseBlockHeader): Promise { throw new Error('GoChaintracksServiceClient.addHeader is not supported by the remote v2 API.') } - async startListening (): Promise { + async startListening(): Promise { await this.getPresentHeight() } - async listening (): Promise { + async listening(): Promise { await this.getPresentHeight() } - async isListening (): Promise { + async isListening(): Promise { try { await this.getPresentHeight() return true @@ -132,18 +153,18 @@ export class GoChaintracksServiceClient implements ChaintracksClientApi { } } - async isSynchronized (): Promise { + async isSynchronized(): Promise { return await this.isListening() } - async subscribeHeaders (listener: HeaderListener): Promise { - return this.subscribe('header', '/tip/stream', (payload) => { + async subscribeHeaders(listener: HeaderListener): Promise { + return this.subscribe('header', '/tip/stream', payload => { listener(payload as BlockHeader) }) } - async subscribeReorgs (listener: ReorgListener): Promise { - return this.subscribe('reorg', '/reorg/stream', (payload) => { + async subscribeReorgs(listener: ReorgListener): Promise { + return this.subscribe('reorg', '/reorg/stream', payload => { const event = payload as { depth?: number oldTip?: BlockHeader @@ -156,7 +177,7 @@ export class GoChaintracksServiceClient implements ChaintracksClientApi { }) } - async unsubscribe (subscriptionId: string): Promise { + async unsubscribe(subscriptionId: string): Promise { const sub = this.subscriptions.get(subscriptionId) if (sub == null) return false this.subscriptions.delete(subscriptionId) @@ -165,52 +186,106 @@ export class GoChaintracksServiceClient implements ChaintracksClientApi { return true } - private async subscribe ( + private async subscribe( type: SseSubscription['type'], path: string, onPayload: (payload: unknown) => void ): Promise { const id = `${type}-${this.nextSubscriptionId++}` const abort = new AbortController() - const done = this.runSse(path, abort.signal, onPayload) + const done = this.runSseWithReconnect(path, abort.signal, onPayload) this.subscriptions.set(id, { id, type, abort, done }) - done.catch(() => {}).finally(() => { - const active = this.subscriptions.get(id) - if (active?.abort === abort) this.subscriptions.delete(id) - }) + done + .catch(() => {}) + .finally(() => { + const active = this.subscriptions.get(id) + if (active?.abort === abort) this.subscriptions.delete(id) + }) return id } - private async runSse (path: string, signal: AbortSignal, onPayload: (payload: unknown) => void): Promise { - const response = await this.fetcher(this.url(path), { - headers: { Accept: 'text/event-stream' }, - signal - }) - if (!response.ok) { - throw new Error(`GET ${this.url(path)} failed ${response.status} ${response.statusText}`) - } - if (response.body == null) { - throw new Error(`GET ${this.url(path)} returned no response body`) + private async runSseWithReconnect( + path: string, + signal: AbortSignal, + onPayload: (payload: unknown) => void + ): Promise { + let failures = 0 + while (!signal.aborted) { + try { + const receivedEvent = await this.runSse(path, signal, onPayload) + failures = receivedEvent ? 0 : failures + 1 + } catch { + if (signal.aborted) return + failures++ + } + const multiplier = Math.min(2 ** Math.max(0, failures - 1), 64) + const delay = Math.min(this.reconnectWaitMsecs * multiplier, this.reconnectWaitMaxMsecs) + await this.waitForReconnect(delay, signal) } + } - const reader = response.body.getReader() - const decoder = new TextDecoder() - let buffer = '' + private async waitForReconnect(msecs: number, signal: AbortSignal): Promise { + if (signal.aborted || msecs <= 0) return + await new Promise(resolve => { + let timeout: ReturnType + const onAbort = () => done() + const done = () => { + clearTimeout(timeout) + signal.removeEventListener('abort', onAbort) + resolve() + } + timeout = setTimeout(done, msecs) + signal.addEventListener('abort', onAbort, { once: true }) + }) + } + + private async runSse(path: string, signal: AbortSignal, onPayload: (payload: unknown) => void): Promise { + const controller = new AbortController() + const onAbort = () => controller.abort() + signal.addEventListener('abort', onAbort, { once: true }) + const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMsecs) + let receivedEvent = false + const observePayload = (payload: unknown) => { + receivedEvent = true + onPayload(payload) + } try { - for (;;) { - const { done, value } = await reader.read() - if (done) break - buffer += decoder.decode(value, { stream: true }) - buffer = this.processSseBuffer(buffer, onPayload) + const response = await this.fetcher(this.url(path), { + headers: { Accept: 'text/event-stream' }, + signal: controller.signal + }) + clearTimeout(timeout) + if (!response.ok) { + throw new Error(`GET ${this.url(path)} failed ${response.status} ${response.statusText}`) + } + if (response.body == null) { + throw new Error(`GET ${this.url(path)} returned no response body`) + } + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + try { + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + buffer = this.processSseBuffer(buffer, observePayload) + } + buffer += decoder.decode() + this.processSseBuffer(`${buffer}\n\n`, observePayload) + } finally { + reader.releaseLock() } - buffer += decoder.decode() - this.processSseBuffer(`${buffer}\n\n`, onPayload) } finally { - reader.releaseLock() + clearTimeout(timeout) + signal.removeEventListener('abort', onAbort) } + return receivedEvent } - private processSseBuffer (buffer: string, onPayload: (payload: unknown) => void): string { + private processSseBuffer(buffer: string, onPayload: (payload: unknown) => void): string { + buffer = buffer.replaceAll('\r\n', '\n') for (;;) { const boundary = buffer.indexOf('\n\n') if (boundary < 0) return buffer @@ -237,44 +312,62 @@ export class GoChaintracksServiceClient implements ChaintracksClientApi { } private async getJsonOrUndefined(path: string): Promise { - const response = await this.fetcher(this.url(path), { headers: { Accept: 'application/json' } }) + const response = await this.fetchWithTimeout(this.url(path), { headers: { Accept: 'application/json' } }) if (response.status === 404) return undefined if (!response.ok) { throw new Error(`GET ${this.url(path)} failed ${response.status} ${response.statusText}`) } - return await response.json() as T + const value = (await response.json()) as unknown + if (value != null && typeof value === 'object' && 'status' in value) { + const envelope = value as { status?: string; value?: T; description?: string } + if (envelope.status === 'success') return envelope.value + if (envelope.status === 'error') throw new Error(envelope.description ?? `GET ${this.url(path)} failed`) + } + return value as T } - private async getBinary (path: string): Promise { - const response = await this.fetcher(this.url(path), { headers: { Accept: 'application/octet-stream' } }) + private async getBinary(path: string): Promise { + const response = await this.fetchWithTimeout(this.url(path), { headers: { Accept: 'application/octet-stream' } }) if (!response.ok) { throw new Error(`GET ${this.url(path)} failed ${response.status} ${response.statusText}`) } return new Uint8Array(await response.arrayBuffer()) } - private url (path: string): string { + private async fetchWithTimeout(url: string, init: RequestInit): Promise { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMsecs) + try { + return await this.fetcher(url, { ...init, signal: controller.signal }) + } finally { + clearTimeout(timeout) + } + } + + private url(path: string): string { return `${this.baseUrl}${path}` } - private normalizeChain (network: string): Chain { - switch (network) { + private normalizeChain(network: string): Chain { + switch (network.trim().toLowerCase()) { case 'main': case 'mainnet': return 'main' case 'test': case 'testnet': return 'test' + case 'stn': + case 'scalingtestnet': + return 'stn' case 'ttn': case 'teratest': case 'teratestnet': return 'ttn' case 'tstn': case 'teranodescalingtestnet': - case 'scalingtestnet': return 'tstn' default: - return this.chain + throw new Error(`Unsupported ChainTracks upstream network '${network}'.`) } } } diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/BulkIngestorChaintracks.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/BulkIngestorChaintracks.ts new file mode 100644 index 000000000..d83719cb8 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/BulkIngestorChaintracks.ts @@ -0,0 +1,84 @@ +import { Chain } from '../../../../sdk' +import { asUint8Array } from '../../../../utility/utilityHelpers.noBuffer' +import { BulkIngestorBaseOptions } from '../Api/BulkIngestorApi' +import { BlockHeader } from '../Api/BlockHeaderApi' +import { ChaintracksClientApi } from '../Api/ChaintracksClientApi' +import { HeightRange, HeightRanges } from '../util/HeightRange' +import { deserializeBlockHeaders } from '../util/blockHeaderUtilities' +import { BulkIngestorBase } from './BulkIngestorBase' + +export interface BulkIngestorChaintracksOptions extends BulkIngestorBaseOptions { + chain: Chain + chaintracks: ChaintracksClientApi + /** Maximum headers requested from the upstream service at once. */ + maxHeadersPerRequest?: number +} + +/** + * Uses a go-chaintracks/Arcade-compatible service as a validated bulk source. + * Retrieved bytes still pass through ChainTracks' local serialization, hash, + * continuity, and genesis checks before storage. + */ +export class BulkIngestorChaintracks extends BulkIngestorBase { + private readonly chaintracks: ChaintracksClientApi + private readonly maxHeadersPerRequest: number + private networkChecked = false + + constructor(options: BulkIngestorChaintracksOptions) { + super(options) + this.chaintracks = options.chaintracks + this.maxHeadersPerRequest = options.maxHeadersPerRequest ?? 1000 + if (!Number.isInteger(this.maxHeadersPerRequest) || this.maxHeadersPerRequest < 1) { + throw new Error('maxHeadersPerRequest must be a positive integer.') + } + } + + override async getPresentHeight(): Promise { + await this.ensureNetwork() + return await this.chaintracks.getPresentHeight() + } + + async fetchHeaders( + _before: HeightRanges, + fetchRange: HeightRange, + bulkRange: HeightRange, + priorLiveHeaders: BlockHeader[] + ): Promise { + if (fetchRange.isEmpty) return priorLiveHeaders + await this.ensureNetwork() + + let liveHeaders = priorLiveHeaders + let height = fetchRange.minHeight + while (height <= fetchRange.maxHeight) { + const requested = Math.min(this.maxHeadersPerRequest, fetchRange.maxHeight - height + 1) + const hex = await this.chaintracks.getHeaders(height, requested) + const bytes = asUint8Array(hex) + if (bytes.length === 0) { + throw new Error(`ChainTracks upstream returned no headers at height ${height}.`) + } + if (bytes.length % 80 !== 0 || bytes.length > requested * 80) { + throw new Error( + `ChainTracks upstream returned ${bytes.length} bytes for ${requested} headers at height ${height}.` + ) + } + const headers = deserializeBlockHeaders(height, bytes) + liveHeaders = await this.storage().addBulkHeaders(headers, bulkRange, liveHeaders) + height += headers.length + if (headers.length < requested && height <= fetchRange.maxHeight) { + throw new Error( + `ChainTracks upstream returned ${headers.length} of ${requested} headers at height ${height - headers.length}.` + ) + } + } + return liveHeaders + } + + private async ensureNetwork(): Promise { + if (this.networkChecked) return + const actual = await this.chaintracks.getChain() + if (actual !== this.chain) { + throw new Error(`ChainTracks upstream network '${actual}' does not match configured chain '${this.chain}'.`) + } + this.networkChecked = true + } +} diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/LiveIngestorChaintracksSSE.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/LiveIngestorChaintracksSSE.ts index 4428a1337..b7ff4b046 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/LiveIngestorChaintracksSSE.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/LiveIngestorChaintracksSSE.ts @@ -12,7 +12,7 @@ export interface LiveIngestorChaintracksSSEOptions extends LiveIngestorBaseOptio * `/chaintracks/v2/tip/stream`, into the local Chaintracks live-ingestor API. */ export class LiveIngestorChaintracksSSE extends LiveIngestorBase { - static createLiveIngestorChaintracksSSEOptions ( + static createLiveIngestorChaintracksSSEOptions( chain: Chain, chaintracks: ChaintracksClientApi ): LiveIngestorChaintracksSSEOptions { @@ -26,26 +26,36 @@ export class LiveIngestorChaintracksSSE extends LiveIngestorBase { private stopped = false private resolveStopped?: () => void - constructor (private readonly options: LiveIngestorChaintracksSSEOptions) { + constructor(private readonly options: LiveIngestorChaintracksSSEOptions) { super(options) } - async getHeaderByHash (hash: string): Promise { + async getHeaderByHash(hash: string): Promise { return await this.options.chaintracks.findHeaderForBlockHash(hash) } - async startListening (liveHeaders: BlockHeader[]): Promise { + async startListening(liveHeaders: BlockHeader[]): Promise { this.stopped = false - this.subscriptionId = await this.options.chaintracks.subscribeHeaders(header => { + const actual = await this.options.chaintracks.getChain() + if (actual !== this.chain) { + throw new Error(`ChainTracks upstream network '${actual}' does not match configured chain '${this.chain}'.`) + } + if (this.stopped) return + const subscriptionId = await this.options.chaintracks.subscribeHeaders(header => { if (!this.stopped) liveHeaders.push(header) }) + if (this.stopped) { + await this.options.chaintracks.unsubscribe(subscriptionId) + return + } + this.subscriptionId = subscriptionId await new Promise(resolve => { this.resolveStopped = resolve if (this.stopped) resolve() }) } - stopListening (): void { + stopListening(): void { this.stopped = true const subscriptionId = this.subscriptionId this.subscriptionId = undefined @@ -54,10 +64,12 @@ export class LiveIngestorChaintracksSSE extends LiveIngestorBase { this.log(`LiveIngestorChaintracksSSE unsubscribe failed: ${e}`) }) } - this.resolveStopped?.() + const resolveStopped = this.resolveStopped + this.resolveStopped = undefined + resolveStopped?.() } - override async shutdown (): Promise { + override async shutdown(): Promise { this.stopListening() } } diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainIngestorWs.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainIngestorWs.ts index ed772d33a..85329db42 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainIngestorWs.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainIngestorWs.ts @@ -156,15 +156,16 @@ export async function WocHeadersBulkListener( let webSocketUrl: string switch (chain) { case 'test': - case 'ttn': - case 'tstn': webSocketUrl = `wss://socket-v2-testnet.whatsonchain.com/websocket/blockheaders/history?from=${fromHeight}&to=${toHeight}` break case 'main': webSocketUrl = `wss://socket-v2.whatsonchain.com/websocket/blockheaders/history?from=${fromHeight}&to=${toHeight}` break + case 'stn': + case 'ttn': + case 'tstn': case 'mock': - throw new Error("WocHeadersBulkListener does not support 'mock' chain.") + throw new Error(`WocHeadersBulkListener does not support '${chain}' chain.`) } const ws = new WebSocket(webSocketUrl) @@ -335,15 +336,16 @@ export async function WocHeadersLiveListener( let webSocketUrl: string switch (chain) { case 'test': - case 'ttn': - case 'tstn': webSocketUrl = 'wss://socket-v2-testnet.whatsonchain.com/websocket/blockHeaders' break case 'main': webSocketUrl = 'wss://socket-v2.whatsonchain.com/websocket/blockHeaders' break + case 'stn': + case 'ttn': + case 'tstn': case 'mock': - throw new Error("WocHeadersLiveListener does not support 'mock' chain.") + throw new Error(`WocHeadersLiveListener does not support '${chain}' chain.`) } function processData(rawData: WebSocket.Data) { diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainServices.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainServices.ts index b9269ba60..68b4beb3a 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainServices.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainServices.ts @@ -4,6 +4,7 @@ import { WhatsOnChain, WocChainInfo } from '../../../providers/WhatsOnChain' import { ChaintracksFetchApi } from '../Api/ChaintracksFetchApi' import { ChaintracksFetch } from '../util/ChaintracksFetch' import { HeightRange } from '../util/HeightRange' +import { wait } from '../../../../utility/utilityHelpers' /** * return true to ignore error, false to close service connection @@ -62,14 +63,14 @@ async function resolveHeaderFileLink( export interface WhatsOnChainServicesOptions { /** - * Which chain is being tracked: main, test, or stn. + * Which chain is being tracked. The public WhatsOnChain fallback is only + * configured automatically for mainnet and testnet. */ chain: Chain /** - * WhatsOnChain.com API Key - * https://docs.taal.com/introduction/get-an-api-key - * If unknown or empty, maximum request rate is limited. - * https://developers.whatsonchain.com/#rate-limits + * Optional WhatsOnChain API key. ChainTracks works without one and limits + * anonymous traffic to the documented public rate. + * https://docs.whatsonchain.com/ */ apiKey?: string /** @@ -88,6 +89,8 @@ export interface WhatsOnChainServicesOptions { * How long chainInfo is considered still valid before updating (msecs). */ chainInfoMsecs: number + /** Minimum interval between keyless API request starts. Defaults below 3 requests/second. */ + minRequestIntervalMsecs?: number } export class WhatsOnChainServices { @@ -98,7 +101,8 @@ export class WhatsOnChainServices { timeout: 30000, userAgent: 'BabbageWhatsOnChainServices', enableCache: true, - chainInfoMsecs: 5000 + chainInfoMsecs: 5000, + minRequestIntervalMsecs: 350 } return options } @@ -106,6 +110,9 @@ export class WhatsOnChainServices { static readonly chainInfo: Array = [] static readonly chainInfoTime: Array = [] static readonly chainInfoMsecs: number[] = [] + static readonly chainInfoPromise: Partial>> = {} + private static requestTail: Promise = Promise.resolve() + private static nextRequestMsecs = 0 chain: Chain woc: WhatsOnChain @@ -115,7 +122,8 @@ export class WhatsOnChainServices { apiKey: this.options.apiKey, timeout: this.options.timeout, userAgent: this.options.userAgent, - enableCache: this.options.enableCache + enableCache: this.options.enableCache, + requestGate: async () => await this.waitForRateLimit() } this.chain = options.chain const chainInfoMsecs = WhatsOnChainServices.chainInfoMsecs as unknown as Record @@ -139,7 +147,18 @@ export class WhatsOnChainServices { update = elapsed > chainInfoMsecs[this.chain]! } if (update) { - chainInfo[this.chain] = await this.woc.getChainInfo() + let pending = WhatsOnChainServices.chainInfoPromise[this.chain] + if (pending == null) { + pending = this.woc.getChainInfo() + WhatsOnChainServices.chainInfoPromise[this.chain] = pending + } + try { + chainInfo[this.chain] = await pending + } finally { + if (WhatsOnChainServices.chainInfoPromise[this.chain] === pending) { + delete WhatsOnChainServices.chainInfoPromise[this.chain] + } + } chainInfoTime[this.chain] = now } if (!chainInfo[this.chain]) throw new Error('Unexpected failure to update chainInfo.') @@ -160,6 +179,7 @@ export class WhatsOnChainServices { */ async getHeaders(fetch?: ChaintracksFetchApi): Promise { fetch ||= new ChaintracksFetch() + await this.waitForRateLimit() const headers = await fetch.fetchJson( `https://api.whatsonchain.com/v1/bsv/${this.chain}/block/headers` ) @@ -171,6 +191,7 @@ export class WhatsOnChainServices { fetch?: ChaintracksFetchApi ): Promise { fetch ||= new ChaintracksFetch() + await this.waitForRateLimit() const files = await fetch.fetchJson( `https://api.whatsonchain.com/v1/bsv/${this.chain}/block/headers/resources` ) @@ -185,6 +206,22 @@ export class WhatsOnChainServices { } return r } + + private async waitForRateLimit(): Promise { + let release!: () => void + const previous = WhatsOnChainServices.requestTail + WhatsOnChainServices.requestTail = new Promise(resolve => { + release = resolve + }) + await previous + try { + const delay = Math.max(0, WhatsOnChainServices.nextRequestMsecs - Date.now()) + if (delay > 0) await wait(delay) + WhatsOnChainServices.nextRequestMsecs = Date.now() + (this.options.minRequestIntervalMsecs ?? 350) + } finally { + release() + } + } } export interface WocGetHeaderByteFileLinks { diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/BulkIngestorChaintracks.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/BulkIngestorChaintracks.test.ts new file mode 100644 index 000000000..389d3f3a6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/BulkIngestorChaintracks.test.ts @@ -0,0 +1,150 @@ +import { ChaintracksClientApi } from '../../Api/ChaintracksClientApi' +import { ChaintracksStorageBase } from '../../Storage/ChaintracksStorageBase' +import { HeightRange } from '../../util/HeightRange' +import { serializeBaseBlockHeader } from '../../util/blockHeaderUtilities' +import { BulkIngestorChaintracks } from '../BulkIngestorChaintracks' + +const toHex = (bytes: number[]): string => bytes.map(byte => byte.toString(16).padStart(2, '0')).join('') + +describe('BulkIngestorChaintracks', () => { + test('rejects invalid batch limits and returns an empty range without contacting the upstream', async () => { + const remote = { getChain: jest.fn() } as unknown as ChaintracksClientApi + expect( + () => + new BulkIngestorChaintracks({ + chain: 'main', + jsonResource: 'mainNetBlockHeaders.json', + chaintracks: remote, + maxHeadersPerRequest: 0 + }) + ).toThrow('maxHeadersPerRequest must be a positive integer') + + const ingestor = new BulkIngestorChaintracks({ + chain: 'main', + jsonResource: 'mainNetBlockHeaders.json', + chaintracks: remote + }) + const prior: any[] = [] + await expect( + ingestor.fetchHeaders( + { bulk: HeightRange.empty, live: HeightRange.empty }, + HeightRange.empty, + HeightRange.empty, + prior + ) + ).resolves.toBe(prior) + expect(remote.getChain).not.toHaveBeenCalled() + }) + + test('fetches bounded header batches and forwards them through local storage validation', async () => { + const headers = [0, 1, 2].map(height => + serializeBaseBlockHeader({ + version: 1, + previousHash: height.toString(16).padStart(64, '0'), + merkleRoot: (height + 1).toString(16).padStart(64, '0'), + time: height, + bits: 0x1d00ffff, + nonce: height + }) + ) + const getHeaders = jest.fn(async (height: number, count: number) => + toHex(headers.slice(height, height + count).flat()) + ) + const remote = { + getChain: jest.fn(async () => 'ttn'), + getPresentHeight: jest.fn(async () => 2), + getHeaders + } as unknown as ChaintracksClientApi + const addBulkHeaders = jest.fn(async (batch, _bulkRange, live) => [...live, ...batch]) + const storage = { addBulkHeaders } as unknown as ChaintracksStorageBase + const ingestor = new BulkIngestorChaintracks({ + chain: 'ttn', + jsonResource: 'ttnNetBlockHeaders.json', + chaintracks: remote, + maxHeadersPerRequest: 2 + }) + await ingestor.setStorage(storage, () => {}) + + await expect(ingestor.getPresentHeight()).resolves.toBe(2) + const result = await ingestor.fetchHeaders( + { bulk: HeightRange.empty, live: HeightRange.empty }, + new HeightRange(0, 2), + new HeightRange(0, 1), + [] + ) + + expect(getHeaders).toHaveBeenNthCalledWith(1, 0, 2) + expect(getHeaders).toHaveBeenNthCalledWith(2, 2, 1) + expect(addBulkHeaders).toHaveBeenCalledTimes(2) + expect(result.map(header => header.height)).toEqual([0, 1, 2]) + }) + + test('rejects an upstream configured for another network', async () => { + const remote = { + getChain: jest.fn(async () => 'test') + } as unknown as ChaintracksClientApi + const ingestor = new BulkIngestorChaintracks({ + chain: 'ttn', + jsonResource: 'ttnNetBlockHeaders.json', + chaintracks: remote + }) + + await expect(ingestor.getPresentHeight()).rejects.toThrow("network 'test' does not match configured chain 'ttn'") + }) + + test('treats an incomplete upstream batch as a source failure', async () => { + const remote = { + getChain: jest.fn(async () => 'main'), + getHeaders: jest.fn(async () => '') + } as unknown as ChaintracksClientApi + const ingestor = new BulkIngestorChaintracks({ + chain: 'main', + jsonResource: 'mainNetBlockHeaders.json', + chaintracks: remote + }) + + await expect( + ingestor.fetchHeaders( + { bulk: HeightRange.empty, live: HeightRange.empty }, + new HeightRange(0, 1), + new HeightRange(0, 1), + [] + ) + ).rejects.toThrow('returned no headers at height 0') + }) + + test('rejects malformed and short non-empty upstream batches', async () => { + const header = serializeBaseBlockHeader({ + version: 1, + previousHash: '00'.repeat(32), + merkleRoot: '11'.repeat(32), + time: 1, + bits: 0x1d00ffff, + nonce: 1 + }) + const getHeaders = jest.fn().mockResolvedValueOnce('00').mockResolvedValueOnce(toHex(header)) + const remote = { + getChain: jest.fn(async () => 'main'), + getHeaders + } as unknown as ChaintracksClientApi + const storage = { + addBulkHeaders: jest.fn(async (headers, _range, live) => [...live, ...headers]) + } as unknown as ChaintracksStorageBase + const ingestor = new BulkIngestorChaintracks({ + chain: 'main', + jsonResource: 'mainNetBlockHeaders.json', + chaintracks: remote, + maxHeadersPerRequest: 2 + }) + await ingestor.setStorage(storage, () => {}) + const args = [ + { bulk: HeightRange.empty, live: HeightRange.empty }, + new HeightRange(0, 1), + new HeightRange(0, 1), + [] + ] as const + + await expect(ingestor.fetchHeaders(...args)).rejects.toThrow('returned 1 bytes for 2 headers') + await expect(ingestor.fetchHeaders(...args)).rejects.toThrow('returned 1 of 2 headers at height 0') + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/LiveIngestorChaintracksSSE.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/LiveIngestorChaintracksSSE.test.ts index 4cc41f223..7cdebdc01 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/LiveIngestorChaintracksSSE.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/LiveIngestorChaintracksSSE.test.ts @@ -14,6 +14,7 @@ describe('LiveIngestorChaintracksSSE', () => { } let listener: any const chaintracks = { + getChain: jest.fn(async () => 'main'), subscribeHeaders: jest.fn(async cb => { listener = cb return 'sub-1' @@ -37,4 +38,57 @@ describe('LiveIngestorChaintracksSSE', () => { expect(chaintracks.unsubscribe).toHaveBeenCalledWith('sub-1') await expect(ingestor.getHeaderByHash(header.hash)).resolves.toEqual(header) }) + + test('does not leak a subscription when shutdown wins the network-check race', async () => { + let resolveNetwork!: (chain: 'main') => void + const network = new Promise<'main'>(resolve => { + resolveNetwork = resolve + }) + const chaintracks = { + getChain: jest.fn(async () => await network), + subscribeHeaders: jest.fn(async () => 'sub-1'), + unsubscribe: jest.fn(async () => true) + } as any + const ingestor = new LiveIngestorChaintracksSSE({ chain: 'main', chaintracks }) + + const listening = ingestor.startListening([]) + ingestor.stopListening() + resolveNetwork('main') + await listening + + expect(chaintracks.subscribeHeaders).not.toHaveBeenCalled() + expect(chaintracks.unsubscribe).not.toHaveBeenCalled() + }) + + test('rejects a mismatched upstream network', async () => { + const chaintracks = { + getChain: jest.fn(async () => 'test'), + subscribeHeaders: jest.fn() + } as any + const ingestor = new LiveIngestorChaintracksSSE({ chain: 'main', chaintracks }) + + await expect(ingestor.startListening([])).rejects.toThrow("network 'test' does not match configured chain 'main'") + expect(chaintracks.subscribeHeaders).not.toHaveBeenCalled() + }) + + test('unsubscribes when shutdown wins the pending subscribe race', async () => { + let resolveSubscription!: (id: string) => void + const subscription = new Promise(resolve => { + resolveSubscription = resolve + }) + const chaintracks = { + getChain: jest.fn(async () => 'main'), + subscribeHeaders: jest.fn(async () => await subscription), + unsubscribe: jest.fn(async () => true) + } as any + const ingestor = new LiveIngestorChaintracksSSE({ chain: 'main', chaintracks }) + + const listening = ingestor.startListening([]) + await new Promise(resolve => setTimeout(resolve, 0)) + ingestor.stopListening() + resolveSubscription('sub-race') + await listening + + expect(chaintracks.unsubscribe).toHaveBeenCalledWith('sub-race') + }) }) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/WhatsOnChainServices.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/WhatsOnChainServices.test.ts index 96f121669..92d7c4fc9 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/WhatsOnChainServices.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Ingest/__tests/WhatsOnChainServices.test.ts @@ -2,6 +2,28 @@ import { WhatsOnChainServices, parseFileLink } from '../WhatsOnChainServices' import { HeightRange } from '../../util/HeightRange' describe('WhatsOnChain header file links', () => { + test('coalesces concurrent chain-height reads into one rate-limited request', async () => { + const options = WhatsOnChainServices.createWhatsOnChainServicesOptions('test') + options.chainInfoMsecs = 0 + options.minRequestIntervalMsecs = 0 + const service = new WhatsOnChainServices(options) + const result = { + chain: 'test', + blocks: 123, + headers: 123, + bestblockhash: '00'.repeat(32), + difficulty: 1, + mediantime: 1, + verificationprogress: 1, + pruned: false, + chainwork: '00'.repeat(32) + } + const getChainInfo = jest.spyOn(service.woc, 'getChainInfo').mockResolvedValue(result) + + await expect(Promise.all([service.getChainTipHeight(), service.getChainTipHeight()])).resolves.toEqual([123, 123]) + expect(getChainInfo).toHaveBeenCalledTimes(1) + }) + test('parses latest and bounded header resources', () => { expect(parseFileLink('https://cdn.example/headers/latest')).toEqual({ range: 'latest', diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts index 603855c02..aedca2855 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts @@ -50,26 +50,59 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { hashToHeaderId: new Map() } - constructor (options: ChaintracksStorageNoDbOptions) { + static readonly stnData: ChaintracksNoDbData = { + chain: 'stn', + liveHeaders: new Map(), + maxHeaderId: 0, + tipHeaderId: 0, + hashToHeaderId: new Map() + } + + static readonly ttnData: ChaintracksNoDbData = { + chain: 'ttn', + liveHeaders: new Map(), + maxHeaderId: 0, + tipHeaderId: 0, + hashToHeaderId: new Map() + } + + static readonly tstnData: ChaintracksNoDbData = { + chain: 'tstn', + liveHeaders: new Map(), + maxHeaderId: 0, + tipHeaderId: 0, + hashToHeaderId: new Map() + } + + constructor(options: ChaintracksStorageNoDbOptions) { super(options) } - override async destroy (): Promise { /* intentional no-op: in-memory storage has no cleanup */ } + override async destroy(): Promise { + /* intentional no-op: in-memory storage has no cleanup */ + } - async getData (): Promise { + async getData(): Promise { switch (this.chain) { case 'main': return ChaintracksStorageNoDb.mainData case 'test': + return ChaintracksStorageNoDb.testData + case 'stn': + return ChaintracksStorageNoDb.stnData case 'ttn': + return ChaintracksStorageNoDb.ttnData case 'tstn': - return ChaintracksStorageNoDb.testData + return ChaintracksStorageNoDb.tstnData default: - throw new WERR_INVALID_PARAMETER('chain', `'main', 'test', 'ttn', or 'tstn'. '${this.chain}' is unsupported.`) + throw new WERR_INVALID_PARAMETER( + 'chain', + `'main', 'test', 'stn', 'ttn', or 'tstn'. '${this.chain}' is unsupported.` + ) } } - override async deleteLiveBlockHeaders (): Promise { + override async deleteLiveBlockHeaders(): Promise { const data = await this.getData() data.liveHeaders.clear() data.maxHeaderId = 0 @@ -77,7 +110,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { data.hashToHeaderId.clear() } - override async deleteOlderLiveBlockHeaders (maxHeight: number): Promise { + override async deleteOlderLiveBlockHeaders(maxHeight: number): Promise { const data = await this.getData() let deletedCount = 0 @@ -85,7 +118,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { for (const [headerId, header] of data.liveHeaders) { if (header.previousHeaderId) { const prevHeader = data.liveHeaders.get(header.previousHeaderId) - if ((prevHeader != null) && prevHeader.height <= maxHeight) { + if (prevHeader != null && prevHeader.height <= maxHeight) { data.liveHeaders.set(headerId, { ...header, previousHeaderId: null }) } } @@ -115,42 +148,42 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { return deletedCount } - override async findChainTipHeader (): Promise { + override async findChainTipHeader(): Promise { const data = await this.getData() const tip = Array.from(data.liveHeaders.values()).find(h => h.isActive && h.isChainTip) if (tip == null) throw new Error('Database contains no active chain tip header.') return tip } - override async findChainTipHeaderOrUndefined (): Promise { + override async findChainTipHeaderOrUndefined(): Promise { const data = await this.getData() return Array.from(data.liveHeaders.values()).find(h => h.isActive && h.isChainTip) } - override async findLiveHeaderForBlockHash (hash: string): Promise { + override async findLiveHeaderForBlockHash(hash: string): Promise { const data = await this.getData() const headerId = data.hashToHeaderId.get(hash) return headerId ? data.liveHeaders.get(headerId) || null : null } - override async findLiveHeaderForHeaderId (headerId: number): Promise { + override async findLiveHeaderForHeaderId(headerId: number): Promise { const data = await this.getData() const header = data.liveHeaders.get(headerId) if (header == null) throw new Error(`HeaderId ${headerId} not found in live header database.`) return header } - override async findLiveHeaderForHeight (height: number): Promise { + override async findLiveHeaderForHeight(height: number): Promise { const data = await this.getData() return Array.from(data.liveHeaders.values()).find(h => h.height === height && h.isActive) || null } - override async findLiveHeaderForMerkleRoot (merkleRoot: string): Promise { + override async findLiveHeaderForMerkleRoot(merkleRoot: string): Promise { const data = await this.getData() return Array.from(data.liveHeaders.values()).find(h => h.merkleRoot === merkleRoot) || null } - override async findLiveHeightRange (): Promise { + override async findLiveHeightRange(): Promise { const data = await this.getData() const activeHeaders = Array.from(data.liveHeaders.values()).filter(h => h.isActive) if (activeHeaders.length === 0) { @@ -161,12 +194,12 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { return new HeightRange(minHeight, maxHeight) } - override async findMaxHeaderId (): Promise { + override async findMaxHeaderId(): Promise { const data = await this.getData() return data.maxHeaderId } - override async liveHeadersForBulk (count: number): Promise { + override async liveHeadersForBulk(count: number): Promise { const data = await this.getData() return Array.from(data.liveHeaders.values()) .filter(h => h.isActive) @@ -174,7 +207,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { .slice(0, count) } - override async getLiveHeaders (range: HeightRange): Promise { + override async getLiveHeaders(range: HeightRange): Promise { if (range.isEmpty) return [] const data = await this.getData() const headers = Array.from(data.liveHeaders.values()) @@ -183,7 +216,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { return headers } - private async insertFirstHeader ( + private async insertFirstHeader( data: ChaintracksNoDbData, header: BlockHeader, result: InsertHeaderResult @@ -191,9 +224,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { if (data.liveHeaders.size !== 0) return false const lastBulkFile = await this.bulkManager.getLastFile() if (lastBulkFile == null) { - throw new WERR_INVALID_OPERATION( - 'bulk headers must exist before first live header can be added' - ) + throw new WERR_INVALID_OPERATION('bulk headers must exist before first live header can be added') } if ( header.previousHash !== lastBulkFile.lastHash || @@ -205,10 +236,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { ...header, headerId: ++data.maxHeaderId, previousHeaderId: null, - chainWork: addWork( - lastBulkFile.lastChainWork, - convertBitsToWork(header.bits) - ), + chainWork: addWork(lastBulkFile.lastChainWork, convertBitsToWork(header.bits)), isChainTip: true, isActive: true } @@ -220,16 +248,14 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { return true } - private findActiveAncestor ( + private findActiveAncestor( data: ChaintracksNoDbData, oneBack: LiveBlockHeader, result: InsertHeaderResult ): LiveBlockHeader | undefined { let activeAncestor = oneBack while (!activeAncestor.isActive) { - const previousHeader = data.liveHeaders.get( - activeAncestor.previousHeaderId! - ) + const previousHeader = data.liveHeaders.get(activeAncestor.previousHeaderId!) if (previousHeader == null) { result.noActiveAncestor = true return undefined @@ -239,7 +265,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { return activeAncestor } - private applyReorganization ( + private applyReorganization( data: ChaintracksNoDbData, oneBack: LiveBlockHeader, activeAncestor: LiveBlockHeader, @@ -249,18 +275,13 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { let headerToDeactivate = Array.from(data.liveHeaders.values()).find( candidate => candidate.isChainTip && candidate.isActive ) - while ( - headerToDeactivate != null && - headerToDeactivate.headerId !== activeAncestor.headerId - ) { + while (headerToDeactivate != null && headerToDeactivate.headerId !== activeAncestor.headerId) { result.deactivatedHeaders.push(headerToDeactivate) data.liveHeaders.set(headerToDeactivate.headerId, { ...headerToDeactivate, isActive: false }) - headerToDeactivate = data.liveHeaders.get( - headerToDeactivate.previousHeaderId! - ) + headerToDeactivate = data.liveHeaders.get(headerToDeactivate.previousHeaderId!) } let headerToActivate = oneBack while (headerToActivate.headerId !== activeAncestor.headerId) { @@ -268,13 +289,11 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { ...headerToActivate, isActive: true }) - headerToActivate = data.liveHeaders.get( - headerToActivate.previousHeaderId! - )! + headerToActivate = data.liveHeaders.get(headerToActivate.previousHeaderId!)! } } - private prepareActiveTip ( + private prepareActiveTip( data: ChaintracksNoDbData, header: BlockHeader, oneBack: LiveBlockHeader, @@ -284,14 +303,13 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase { const activeAncestor = this.findActiveAncestor(data, oneBack, result) if (activeAncestor == null) return false if (!(oneBack.isActive && oneBack.isChainTip)) { - result.reorgDepth = - Math.min(result.priorTip!.height, header.height) - activeAncestor.height + result.reorgDepth = Math.min(result.priorTip!.height, header.height) - activeAncestor.height } this.applyReorganization(data, oneBack, activeAncestor, result) return true } - override async insertHeader (header: BlockHeader): Promise { + override async insertHeader(header: BlockHeader): Promise { const data = await this.getData() const r = createInsertHeaderResult() diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/__tests/ChaintracksStorageNoDb.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/__tests/ChaintracksStorageNoDb.test.ts index e95a3f41b..ad8a66e4a 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/__tests/ChaintracksStorageNoDb.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Storage/__tests/ChaintracksStorageNoDb.test.ts @@ -100,4 +100,84 @@ describe('ChaintracksStorageNoDb insertHeader compatibility', () => { noPrev: true }) }) + + test('keeps in-memory state isolated for every supported network', async () => { + const chains = ['main', 'test', 'stn', 'ttn', 'tstn'] as const + const stores = chains.map( + chain => new ChaintracksStorageNoDb(ChaintracksStorageBase.createStorageBaseOptions(chain)) + ) + const datasets = await Promise.all(stores.map(async store => await store.getData())) + + for (const [index, data] of datasets.entries()) { + expect(data.chain).toBe(chains[index]) + for (const [otherIndex, other] of datasets.entries()) { + if (otherIndex !== index) { + expect(data).not.toBe(other) + expect(data.liveHeaders).not.toBe(other.liveHeaders) + expect(data.hashToHeaderId).not.toBe(other.hashToHeaderId) + } + } + } + }) + + test('rejects mock storage before it can share a public-network data set', async () => { + const mockStorage = new ChaintracksStorageNoDb(ChaintracksStorageBase.createStorageBaseOptions('mock')) + + await expect(mockStorage.getData()).rejects.toThrow("'mock' is unsupported") + }) + + test('disconnects surviving headers from live ancestors that are pruned', async () => { + const bulkTipHash = 'a0'.repeat(32) + jest.spyOn(storage.bulkManager, 'getLastFile').mockResolvedValue({ + chain: 'main', + fileName: 'test.headers', + firstHeight: 0, + count: 100, + prevChainWork: '00'.repeat(32), + lastChainWork: '01'.repeat(32), + prevHash: '00'.repeat(32), + lastHash: bulkTipHash, + fileHash: null + }) + const first = makeHeader(100, 'b', bulkTipHash) + const second = makeHeader(101, 'c', first.hash) + + await storage.insertHeader(first) + await storage.insertHeader(second) + + await expect(storage.deleteOlderLiveBlockHeaders(100)).resolves.toBe(1) + await expect(storage.findLiveHeaderForBlockHash(first.hash)).resolves.toBeNull() + await expect(storage.findLiveHeaderForBlockHash(second.hash)).resolves.toMatchObject({ + previousHeaderId: null + }) + }) + + test('tolerates a surviving header whose live ancestor is already absent', async () => { + const bulkTipHash = 'a0'.repeat(32) + jest.spyOn(storage.bulkManager, 'getLastFile').mockResolvedValue({ + chain: 'main', + fileName: 'test.headers', + firstHeight: 0, + count: 100, + prevChainWork: '00'.repeat(32), + lastChainWork: '01'.repeat(32), + prevHash: '00'.repeat(32), + lastHash: bulkTipHash, + fileHash: null + }) + const first = makeHeader(100, 'b', bulkTipHash) + const second = makeHeader(101, 'c', first.hash) + + await storage.insertHeader(first) + await storage.insertHeader(second) + const persistedFirst = await storage.findLiveHeaderForBlockHash(first.hash) + if (persistedFirst == null) throw new Error('Expected the first live header to be persisted') + const data = await storage.getData() + data.liveHeaders.delete(persistedFirst.headerId) + + await expect(storage.deleteOlderLiveBlockHeaders(99)).resolves.toBe(0) + await expect(storage.findLiveHeaderForBlockHash(second.hash)).resolves.toMatchObject({ + previousHeaderId: persistedFirst.headerId + }) + }) }) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksClientApi.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksClientApi.test.ts index d9f0ea318..7cf4056db 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksClientApi.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksClientApi.test.ts @@ -25,7 +25,7 @@ const fixtureRoot = './src/services/chaintracker/chaintracks/__tests/data/cdnTes const fixtureCdnUrl = 'https://fixture.invalid/blockheaders/' describe('ChaintracksClientApi deterministic contract', () => { - const clients: Array<{ client: ChaintracksClientApi, chain: Chain }> = [] + const clients: Array<{ client: ChaintracksClientApi; chain: Chain }> = [] let localService: ChaintracksService let localChaintracks: Chaintracks let firstTip: BlockHeader @@ -46,7 +46,8 @@ describe('ChaintracksClientApi deterministic contract', () => { 2, 100, 100, - 36 + 36, + { disableChaintracks: true } ) options.logging = () => {} @@ -81,16 +82,9 @@ describe('ChaintracksClientApi deterministic contract', () => { // Each Jest worker has a separate process ID, avoiding collisions when // package tests execute in parallel. await localService.startJsonRpcServer(30000 + (process.pid % 10000)) - const localServiceClient = new ChaintracksServiceClient( - chain, - `http://localhost:${localService.port}`, - {} - ) + const localServiceClient = new ChaintracksServiceClient(chain, `http://localhost:${localService.port}`, {}) - clients.push( - { client: localServiceClient, chain }, - { client: localChaintracks, chain } - ) + clients.push({ client: localServiceClient, chain }, { client: localChaintracks, chain }) firstTip = await clients[0].client.findChainTipHeader() }) @@ -215,12 +209,12 @@ describe('ChaintracksClientApi deterministic contract', () => { }) class FixtureFetch { - constructor ( + constructor( private readonly filesInfo: BulkHeaderFilesInfo, private readonly fileData: Map ) {} - async fetchJson (url: string): Promise { + async fetchJson(url: string): Promise { if (url.endsWith('mainNetBlockHeaders.json')) { return { ...this.filesInfo, @@ -235,7 +229,7 @@ class FixtureFetch { throw new Error(`Unexpected fixture JSON request: ${url}`) } - async download (url: string): Promise { + async download(url: string): Promise { const requestedName = url.split('/').at(-1)! const fileName = requestedName === '400_499_headers' ? 'mainNet_4.headers' : requestedName const data = this.fileData.get(fileName) @@ -243,7 +237,7 @@ class FixtureFetch { return data } - pathJoin (baseUrl: string, subpath: string): string { + pathJoin(baseUrl: string, subpath: string): string { let baseEnd = baseUrl.length while (baseEnd > 0 && baseUrl[baseEnd - 1] === '/') baseEnd-- @@ -254,7 +248,7 @@ class FixtureFetch { } } -async function loadFixtureChain (): Promise<{ +async function loadFixtureChain(): Promise<{ filesInfo: BulkHeaderFilesInfo fileData: Map headers: BlockHeader[] @@ -273,7 +267,7 @@ async function loadFixtureChain (): Promise<{ return { filesInfo, fileData, headers } } -function toWocHeader (header: BlockHeader): WocGetHeadersHeader { +function toWocHeader(header: BlockHeader): WocGetHeadersHeader { return { hash: header.hash, confirmations: 1, diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksConfiguration.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksConfiguration.test.ts index 11f9e54a1..efaf2b06c 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksConfiguration.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/ChaintracksConfiguration.test.ts @@ -1,6 +1,7 @@ import type { Knex } from 'knex' import type { ChaintracksFetchApi } from '../Api/ChaintracksFetchApi' +import type { ChaintracksClientApi } from '../Api/ChaintracksClientApi' import { Chaintracks } from '../Chaintracks' import { createAndStartDefaultChaintracks, @@ -120,6 +121,54 @@ describe('Chaintracks configuration compatibility', () => { await (defaultKnexOptions.storage as ChaintracksStorageKnex).shutdown() }) + test('adds a credential-free remote bulk/live source without changing legacy source order', () => { + const remote = { getChain: jest.fn(async () => 'ttn') } as unknown as ChaintracksClientApi + const sources = { chaintracks: remote, remoteMaxHeadersPerRequest: 250 } + const options = createDefaultNoDbChaintracksOptions('ttn', ...customTail, sources) + const params = resolveDefaultChaintracksArguments(['ttn', ...customTail, sources]) + + expect(toDefaultChaintracksArguments(params)).toEqual(['ttn', ...customTail, sources]) + expect(options.bulkIngestors.map(source => source.constructor.name)).toEqual([ + 'BulkIngestorCDNBabbage', + 'BulkIngestorChaintracks' + ]) + expect(options.liveIngestors.map(source => source.constructor.name)).toEqual(['LiveIngestorChaintracksSSE']) + }) + + test.each(['main', 'test', 'ttn'] as const)( + '%s includes a credential-free public Arcade source by default', + chain => { + const options = createDefaultNoDbChaintracksOptions(chain) + expect(options.bulkIngestors.map(source => source.constructor.name)).toContain('BulkIngestorChaintracks') + expect(options.liveIngestors.map(source => source.constructor.name)).toContain('LiveIngestorChaintracksSSE') + } + ) + + test.each(['stn', 'tstn'] as const)('%s requires an explicit remote live source', chain => { + expect(() => createDefaultNoDbChaintracksOptions(chain)).toThrow( + `ChainTracks ${chain} requires at least one bulk and live source` + ) + }) + + test('allows the public Arcade default to be disabled explicitly', () => { + const options = createDefaultNoDbChaintracksOptions( + 'main', + '', + 100000, + 2, + undefined, + 'https://cdn.projectbabbage.com/blockheaders/', + 2000, + 400, + 500, + 400, + 36, + { disableChaintracks: true } + ) + expect(options.bulkIngestors.map(source => source.constructor.name)).not.toContain('BulkIngestorChaintracks') + expect(options.liveIngestors.map(source => source.constructor.name)).not.toContain('LiveIngestorChaintracksSSE') + }) + test('starts shared and Knex configurations with exact resolved metadata', async () => { const available = Promise.resolve() const makeAvailable = jest.spyOn(Chaintracks.prototype, 'makeAvailable').mockReturnValue(available) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/GoChaintracksServiceClient.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/GoChaintracksServiceClient.test.ts index 8eb6eef76..e7ca6c1e2 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/GoChaintracksServiceClient.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/GoChaintracksServiceClient.test.ts @@ -1,23 +1,23 @@ import { GoChaintracksServiceClient } from '../GoChaintracksServiceClient' -function jsonResponse (data: unknown, status = 200): Response { +function jsonResponse(data: unknown, status = 200): Response { return new Response(JSON.stringify(data), { status, headers: { 'Content-Type': 'application/json' } }) } -function binaryResponse (data: Uint8Array): Response { +function binaryResponse(data: Uint8Array): Response { return new Response(data, { status: 200, headers: { 'Content-Type': 'application/octet-stream' } }) } -function sseResponse (events: string[]): Response { +function sseResponse(events: string[]): Response { const encoder = new TextEncoder() const stream = new ReadableStream({ - start (controller) { + start(controller) { for (const event of events) controller.enqueue(encoder.encode(event)) controller.close() } @@ -29,6 +29,59 @@ function sseResponse (events: string[]): Response { } describe('GoChaintracksServiceClient', () => { + test('rejects timeout settings that could create an unbounded reconnect loop', () => { + expect( + () => + new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + reconnectWaitMsecs: 0 + }) + ).toThrow('reconnectWaitMsecs must be a positive integer') + expect( + () => + new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + reconnectWaitMsecs: 100, + reconnectWaitMaxMsecs: 10 + }) + ).toThrow('reconnectWaitMaxMsecs must be greater than or equal') + }) + + test('unwraps the legacy service envelope while accepting raw go-chaintracks values', async () => { + const fetchMock = jest.fn(async (url: string) => { + if (url.endsWith('/network')) return jsonResponse({ status: 'success', value: 'teratestnet' }) + if (url.endsWith('/height')) return jsonResponse({ status: 'success', value: { height: 44 } }) + return jsonResponse({ status: 'error', description: 'missing' }) + }) as unknown as typeof fetch + const client = new GoChaintracksServiceClient('ttn', 'https://chaintracks.example/v2', { fetch: fetchMock }) + + await expect(client.getChain()).resolves.toBe('ttn') + await expect(client.getPresentHeight()).resolves.toBe(44) + }) + + test('accepts raw height and every supported upstream network alias while rejecting unknown networks', async () => { + const client = new GoChaintracksServiceClient('main', 'https://chaintracks.example/v2', { + fetch: jest.fn(async (url: string) => + url.endsWith('/height') ? jsonResponse(7) : jsonResponse('mainnet') + ) as unknown as typeof fetch + }) + await expect(client.getPresentHeight()).resolves.toBe(7) + await expect(client.getChain()).resolves.toBe('main') + + for (const [alias, chain] of [ + ['scalingtestnet', 'stn'], + ['teranodescalingtestnet', 'tstn'] + ] as const) { + const aliasClient = new GoChaintracksServiceClient(chain, 'https://chaintracks.example/v2', { + fetch: jest.fn(async () => jsonResponse(alias)) as unknown as typeof fetch + }) + await expect(aliasClient.getChain()).resolves.toBe(chain) + } + + const unknown = new GoChaintracksServiceClient('main', 'https://chaintracks.example/v2', { + fetch: jest.fn(async () => jsonResponse('unknownnet')) as unknown as typeof fetch + }) + await expect(unknown.getChain()).rejects.toThrow("Unsupported ChainTracks upstream network 'unknownnet'") + }) + test('reads go-chaintracks v2 height, tip, headers, and hash lookups', async () => { const tip = { version: 1, @@ -83,7 +136,9 @@ describe('GoChaintracksServiceClient', () => { return jsonResponse({ error: 'not found' }, 404) }) as unknown as typeof fetch const client = new GoChaintracksServiceClient('main', 'https://arcade.example.com/chaintracks/v2', { - fetch: fetchMock + fetch: fetchMock, + reconnectWaitMsecs: 10000, + reconnectWaitMaxMsecs: 10000 }) const headers: unknown[] = [] @@ -91,6 +146,95 @@ describe('GoChaintracksServiceClient', () => { await new Promise(resolve => setTimeout(resolve, 0)) expect(headers).toEqual([tip]) + expect(await client.unsubscribe(id)).toBe(true) expect(await client.unsubscribe(id)).toBe(false) }) + + test('reconnects a closed SSE stream and accepts CRLF framing', async () => { + const first = { + version: 1, + previousHash: '00'.repeat(32), + merkleRoot: '11'.repeat(32), + time: 1, + bits: 2, + nonce: 3, + height: 1, + hash: '22'.repeat(32) + } + const second = { ...first, height: 2, hash: '33'.repeat(32) } + let requests = 0 + const fetchMock = jest.fn(async () => { + requests++ + const event = requests === 1 ? first : second + return sseResponse([`data: ${JSON.stringify(event)}\r\n\r\n`]) + }) as unknown as typeof fetch + const client = new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + fetch: fetchMock, + reconnectWaitMsecs: 1, + reconnectWaitMaxMsecs: 1 + }) + const received: unknown[] = [] + const id = await client.subscribeHeaders(header => received.push(header)) + for (let attempt = 0; received.length < 2 && attempt < 20; attempt++) { + await new Promise(resolve => setTimeout(resolve, 2)) + } + + expect(received.slice(0, 2)).toEqual([first, second]) + expect(fetchMock.mock.calls.length).toBeGreaterThanOrEqual(2) + await expect(client.unsubscribe(id)).resolves.toBe(true) + }) + + test('reconnects after a stream request fails and delivers reorg events', async () => { + const oldTip = { height: 1, hash: '11'.repeat(32) } + const newTip = { height: 2, hash: '22'.repeat(32) } + let requests = 0 + const fetchMock = jest.fn(async () => { + requests++ + if (requests === 1) throw new Error('temporary stream failure') + return sseResponse([`data: ${JSON.stringify({ depth: 1, oldTip, newTip, deactivatedHeaders: [oldTip] })}\n\n`]) + }) as unknown as typeof fetch + const client = new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + fetch: fetchMock, + reconnectWaitMsecs: 1, + reconnectWaitMaxMsecs: 1 + }) + const listener = jest.fn() + const id = await client.subscribeReorgs(listener) + for (let attempt = 0; listener.mock.calls.length === 0 && attempt < 20; attempt++) { + await new Promise(resolve => setTimeout(resolve, 2)) + } + + expect(listener).toHaveBeenCalledWith(1, oldTip, newTip, [oldTip]) + expect(fetchMock.mock.calls.length).toBeGreaterThanOrEqual(2) + await expect(client.unsubscribe(id)).resolves.toBe(true) + }) + + test('reports SSE response failures and legacy error envelopes', async () => { + const failed = new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + fetch: jest.fn( + async () => new Response(null, { status: 503, statusText: 'Unavailable' }) + ) as unknown as typeof fetch + }) + await expect((failed as any).runSse('/tip/stream', new AbortController().signal, () => {})).rejects.toThrow( + 'failed 503 Unavailable' + ) + + const bodyless = new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + fetch: jest.fn(async () => ({ ok: true, status: 200, statusText: 'OK', body: null })) as unknown as typeof fetch + }) + await expect((bodyless as any).runSse('/tip/stream', new AbortController().signal, () => {})).rejects.toThrow( + 'returned no response body' + ) + + const envelope = new GoChaintracksServiceClient('main', 'https://arcade.example/v2', { + fetch: jest.fn(async () => + jsonResponse({ status: 'error', description: 'upstream rejected lookup' }) + ) as unknown as typeof fetch + }) + await expect(envelope.findHeaderForHeight(4)).rejects.toThrow('upstream rejected lookup') + + const aborted = new AbortController() + aborted.abort() + await expect((envelope as any).waitForReconnect(1, aborted.signal)).resolves.toBeUndefined() + }) }) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/bulkIngestorFailures.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/bulkIngestorFailures.test.ts index 3a39285bc..0871183d8 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/bulkIngestorFailures.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/__tests/bulkIngestorFailures.test.ts @@ -3,6 +3,234 @@ import { HeightRange } from '../util/HeightRange' import { wait } from '../../../../utility/utilityHelpers' describe('Chaintracks bulk ingestor failure handling', () => { + const liveIngestor = { + setStorage: async () => {}, + startListening: async () => {}, + getHeaderByHash: async () => undefined, + shutdown: async () => {} + } + + test('requires storage and at least one source for each ingestion role', () => { + const storage = { log: () => {} } as any + const bulk = { getPresentHeight: async () => 1 } as any + expect( + () => + new Chaintracks({ + chain: 'main', + storage: undefined as any, + bulkIngestors: [bulk], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false + }) + ).toThrow('storage is required') + expect( + () => + new Chaintracks({ + chain: 'main', + storage, + bulkIngestors: [], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false + }) + ).toThrow('At least one bulk ingestor is required') + expect( + () => + new Chaintracks({ + chain: 'main', + storage, + bulkIngestors: [bulk], + liveIngestors: [], + addLiveRecursionLimit: 36, + readonly: false + }) + ).toThrow('At least one live ingestor is required') + }) + + test('falls through failed present-height sources and records their health', async () => { + const failed = { + getPresentHeight: jest.fn(async () => { + throw new Error('CDN unavailable') + }) + } + const healthy = { + getPresentHeight: jest.fn(async () => 321) + } + const storage = { + log: () => {}, + getAvailableHeightRanges: async () => ({ bulk: HeightRange.empty, live: HeightRange.empty }) + } + const chaintracks = new Chaintracks({ + chain: 'main', + storage: storage as any, + bulkIngestors: [failed as any, healthy as any], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + + await expect(chaintracks.getPresentHeight()).resolves.toBe(321) + expect(failed.getPresentHeight).toHaveBeenCalledTimes(1) + expect(healthy.getPresentHeight).toHaveBeenCalledTimes(1) + expect(Array.from((chaintracks as any).sourceStatus.values())).toEqual( + expect.arrayContaining([ + expect.objectContaining({ role: 'bulk', state: 'degraded', error: 'CDN unavailable' }), + expect.objectContaining({ role: 'bulk', state: 'healthy' }) + ]) + ) + }) + + test('uses the locally validated height when every external provider is unavailable', async () => { + const failed = { + getPresentHeight: jest.fn(async () => { + throw new Error('upstream unavailable') + }) + } + const storage = { + log: () => {}, + getAvailableHeightRanges: async () => ({ bulk: new HeightRange(0, 400), live: new HeightRange(401, 420) }) + } + const chaintracks = new Chaintracks({ + chain: 'main', + storage: storage as any, + bulkIngestors: [failed as any], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + + await expect(chaintracks.getPresentHeight()).resolves.toBe(420) + }) + + test('uses a stale last-good height when providers fail and reports source exhaustion otherwise', async () => { + const unavailable = { getPresentHeight: jest.fn(async () => undefined) } + const emptyStorage = { + log: () => {}, + getAvailableHeightRanges: jest.fn(async () => ({ bulk: HeightRange.empty, live: HeightRange.empty })) + } + const withLastGood = new Chaintracks({ + chain: 'main', + storage: emptyStorage as any, + bulkIngestors: [unavailable as any], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + ;(withLastGood as any).lastPresentHeight = 88 + ;(withLastGood as any).lastPresentHeightMsecs = 0 + await expect(withLastGood.getPresentHeight()).resolves.toBe(88) + + const exhausted = new Chaintracks({ + chain: 'main', + storage: emptyStorage as any, + bulkIngestors: [unavailable as any], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + await expect(exhausted.getPresentHeight()).rejects.toThrow( + 'No present-height source or locally validated headers are available' + ) + + const unreadable = new Chaintracks({ + chain: 'main', + storage: { + log: () => {}, + getAvailableHeightRanges: async () => { + throw new Error('local storage unavailable') + } + } as any, + bulkIngestors: [unavailable as any], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + await expect(unreadable.getPresentHeight()).rejects.toThrow( + 'No present-height source or locally validated headers are available' + ) + }) + + test('continues missing-header lookup with the next live source after a failure', async () => { + const header = { + version: 1, + previousHash: '0'.repeat(64), + merkleRoot: '1'.repeat(64), + time: 1, + bits: 1, + nonce: 1, + height: 1, + hash: '2'.repeat(64) + } + const failed = { + getHeaderByHash: jest.fn(async () => { + throw new Error('primary live source unavailable') + }) + } + const healthy = { + getHeaderByHash: jest.fn(async () => header) + } + const chaintracks = new Chaintracks({ + chain: 'main', + storage: { log: () => {} } as any, + bulkIngestors: [{ getPresentHeight: async () => 1 } as any], + liveIngestors: [failed as any, healthy as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + + await expect((chaintracks as any).getMissingBlockHeader(header.hash)).resolves.toBe(header) + expect(failed.getHeaderByHash).toHaveBeenCalledWith(header.hash) + expect(healthy.getHeaderByHash).toHaveBeenCalledWith(header.hash) + expect(Array.from((chaintracks as any).sourceStatus.values())).toEqual( + expect.arrayContaining([ + expect.objectContaining({ role: 'live', state: 'degraded', error: 'primary live source unavailable' }), + expect.objectContaining({ role: 'live', state: 'healthy' }) + ]) + ) + }) + + test('continues initial bulk synchronization with the next source after a failure', async () => { + const initialRanges = { bulk: new HeightRange(0, 100), live: HeightRange.empty } + const first = { + synchronize: jest.fn(async () => { + throw new Error('primary unavailable') + }) + } + const second = { + synchronize: jest.fn(async () => ({ + liveHeaders: [], + liveRange: HeightRange.empty, + done: true, + log: '' + })) + } + const storage = { + log: () => {}, + getAvailableHeightRanges: async () => initialRanges + } + const chaintracks = new Chaintracks({ + chain: 'main', + storage: storage as any, + bulkIngestors: [first as any, second as any], + liveIngestors: [liveIngestor as any], + addLiveRecursionLimit: 36, + readonly: false, + logging: () => {} + }) + + await expect((chaintracks as any).syncBulkStorageNoLock(101, initialRanges)).resolves.toBeUndefined() + expect(first.synchronize).toHaveBeenCalledTimes(1) + expect(second.synchronize).toHaveBeenCalledTimes(1) + expect((chaintracks as any).startupError).toBeNull() + }) + test('does not loop indefinitely when a bulk ingestor keeps returning incomplete live headers', async () => { const initialRanges = { bulk: new HeightRange(0, 100), diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/configureChaintracksIngestors.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/configureChaintracksIngestors.ts index c8387c533..e83c8bd61 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/configureChaintracksIngestors.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/configureChaintracksIngestors.ts @@ -9,6 +9,24 @@ import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN' import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices' import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager' import { ChaintracksFetch } from './util/ChaintracksFetch' +import { ChaintracksClientApi } from './Api/ChaintracksClientApi' +import { BulkIngestorChaintracks } from './Ingest/BulkIngestorChaintracks' +import { LiveIngestorChaintracksSSE } from './Ingest/LiveIngestorChaintracksSSE' +import { GoChaintracksServiceClient } from './GoChaintracksServiceClient' +import { publicArcadeUrl } from '../../networkConfig' + +export interface ChaintracksSourceOptions { + /** Preferred go-chaintracks or Arcade source. */ + chaintracks?: ChaintracksClientApi + /** Disable the credential-free public Arcade default. */ + disableChaintracks?: boolean + /** Maximum number of headers requested from the remote source at once. */ + remoteMaxHeadersPerRequest?: number + /** Disable the configured CDN source without changing its URL. */ + disableCdn?: boolean + /** Disable the keyless WhatsOnChain fallback on mainnet/testnet. */ + disableWhatsOnChain?: boolean +} export type ChaintracksArgumentsTail = [ whatsonchainApiKey?: string, @@ -20,7 +38,8 @@ export type ChaintracksArgumentsTail = [ reorgHeightThreshold?: number, bulkMigrationChunkSize?: number, batchInsertLimit?: number, - addLiveRecursionLimit?: number + addLiveRecursionLimit?: number, + sources?: ChaintracksSourceOptions ] export type DefaultChaintracksArguments = [chain: Chain, ...options: ChaintracksArgumentsTail] @@ -35,6 +54,7 @@ export interface ChaintracksIngestorParams { fetch: ChaintracksFetchApi cdnUrl: string addLiveRecursionLimit: number + sources: ChaintracksSourceOptions } export interface ResolvedDefaultChaintracksParams extends ChaintracksIngestorParams { @@ -54,7 +74,7 @@ export interface CreatedChaintracks } -export function resolveDefaultChaintracksArguments ( +export function resolveDefaultChaintracksArguments( args: DefaultChaintracksArguments ): ResolvedDefaultChaintracksParams { const [ @@ -63,12 +83,13 @@ export function resolveDefaultChaintracksArguments ( maxPerFile = 100000, maxRetained = 2, fetch = new ChaintracksFetch(), - cdnUrl = 'https://cdn.projectbabbage.com/blockheaders/', + cdnUrl = chain === 'main' || chain === 'test' ? 'https://cdn.projectbabbage.com/blockheaders/' : '', liveHeightThreshold = 2000, reorgHeightThreshold = 400, bulkMigrationChunkSize = 500, batchInsertLimit = 400, - addLiveRecursionLimit = 36 + addLiveRecursionLimit = 36, + sources = {} ] = args return { @@ -82,14 +103,13 @@ export function resolveDefaultChaintracksArguments ( reorgHeightThreshold, bulkMigrationChunkSize, batchInsertLimit, - addLiveRecursionLimit + addLiveRecursionLimit, + sources } } -export function toDefaultChaintracksArguments ( - params: ResolvedDefaultChaintracksParams -): DefaultChaintracksArguments { - return [ +export function toDefaultChaintracksArguments(params: ResolvedDefaultChaintracksParams): DefaultChaintracksArguments { + const args: DefaultChaintracksArguments = [ params.chain, params.whatsonchainApiKey, params.maxPerFile, @@ -102,11 +122,14 @@ export function toDefaultChaintracksArguments ( params.batchInsertLimit, params.addLiveRecursionLimit ] + // Preserve the exact pre-resiliency positional tuple when no source options + // were supplied. This keeps wrappers that inspect or forward arguments + // byte-for-byte compatible while allowing the new options to be appended. + if (Object.keys(params.sources).length > 0) args.push(params.sources) + return args } -export function createDefaultBulkFileDataManager ( - params: ResolvedDefaultChaintracksParams -): BulkFileDataManager { +export function createDefaultBulkFileDataManager(params: ResolvedDefaultChaintracksParams): BulkFileDataManager { const options: BulkFileDataManagerOptions = { chain: params.chain, fetch: params.fetch, @@ -117,9 +140,7 @@ export function createDefaultBulkFileDataManager ( return new BulkFileDataManager(options) } -export function createDefaultChaintracksStorageOptions ( - params: ResolvedDefaultChaintracksParams -) { +export function createDefaultChaintracksStorageOptions(params: ResolvedDefaultChaintracksParams) { return { chain: params.chain, bulkFileDataManager: createDefaultBulkFileDataManager(params), @@ -130,7 +151,7 @@ export function createDefaultChaintracksStorageOptions ( } } -export function startChaintracks ( +export function startChaintracks( params: ResolvedDefaultChaintracksParams, options: ChaintracksOptions ): CreatedChaintracks { @@ -145,7 +166,7 @@ export function startChaintracks } } -export function createAndStartDefaultChaintracks ( +export function createAndStartDefaultChaintracks( args: DefaultChaintracksArguments, createOptions: (...args: DefaultChaintracksArguments) => ChaintracksOptions ): CreatedChaintracks { @@ -160,11 +181,11 @@ export function createAndStartDefaultChaintracks { + const expected: Record, string> = { + main: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', + test: '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943', + stn: '6b38bdbcd73a19f7889d23e1fa6166a9de71affceca60ca3bb1b28af8135c594', + ttn: '000000000499eabba0a88f5b3747231c74b9191c1a4a04b2c2ea817976b7776d', + tstn: '000000005d221c0e023cb56b5682cf094f32cd959958b40bc931e5797cae706c' + } + + test.each(Object.entries(expected) as Array<[Exclude, string]>)( + '%s uses its exact serialized genesis header', + (chain, hash) => { + const bytes = genesisBuffer(chain) + expect(bytes).toHaveLength(80) + expect(blockHash(bytes)).toBe(hash) + expect(genesisHeader(chain).hash).toBe(hash) + expect(() => validateGenesisHeader(Uint8Array.from(bytes), chain)).not.toThrow() + } + ) + + test('Teranode test and scaling-test networks have distinct genesis headers', () => { + expect(genesisBuffer('ttn')).not.toEqual(genesisBuffer('test')) + expect(genesisBuffer('tstn')).not.toEqual(genesisBuffer('test')) + expect(genesisBuffer('tstn')).not.toEqual(genesisBuffer('ttn')) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/util/blockHeaderUtilities.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/util/blockHeaderUtilities.ts index aa16f12fd..0e423ea3b 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/util/blockHeaderUtilities.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/util/blockHeaderUtilities.ts @@ -468,8 +468,6 @@ export function genesisHeader(chain: Chain): BlockHeader { hash: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' } case 'test': - case 'ttn': - case 'tstn': return { version: 1, previousHash: '0000000000000000000000000000000000000000000000000000000000000000', @@ -480,6 +478,41 @@ export function genesisHeader(chain: Chain): BlockHeader { height: 0, hash: '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943' } + case 'stn': + return { + version: 1, + previousHash: '0000000000000000000000000000000000000000000000000000000000000000', + merkleRoot: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + time: 1296688602, + bits: 486604799, + nonce: 173779992, + height: 0, + // go-chaincfg v1.6.1 retains a stale StnParams.GenesisHash literal. + // This is the hash of the serialized stnGenesisBlock header itself. + hash: '6b38bdbcd73a19f7889d23e1fa6166a9de71affceca60ca3bb1b28af8135c594' + } + case 'ttn': + return { + version: 1, + previousHash: '0000000000000000000000000000000000000000000000000000000000000000', + merkleRoot: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + time: 1755606836, + bits: 486604799, + nonce: 1092578460, + height: 0, + hash: '000000000499eabba0a88f5b3747231c74b9191c1a4a04b2c2ea817976b7776d' + } + case 'tstn': + return { + version: 1, + previousHash: '0000000000000000000000000000000000000000000000000000000000000000', + merkleRoot: '64452e5b25c65e492ad6a4f5ce9f427ca986626c28315d88de920d66e28cc98f', + time: 1782864000, + bits: 486604799, + nonce: 1780488216, + height: 0, + hash: '000000005d221c0e023cb56b5682cf094f32cd959958b40bc931e5797cae706c' + } case 'mock': throw new Error("genesisHeader does not support 'mock' chain. Mock chain generates its own genesis block.") } diff --git a/packages/wallet/wallet-toolbox/src/services/createDefaultWalletServicesOptions.ts b/packages/wallet/wallet-toolbox/src/services/createDefaultWalletServicesOptions.ts index 33a3b6ffa..90433f8bd 100644 --- a/packages/wallet/wallet-toolbox/src/services/createDefaultWalletServicesOptions.ts +++ b/packages/wallet/wallet-toolbox/src/services/createDefaultWalletServicesOptions.ts @@ -3,9 +3,47 @@ import { WalletServicesOptions } from '../sdk/WalletServices.interfaces' import { randomBytesHex } from '../utility/utilityHelpers' import { ChaintracksClientApi } from './chaintracker/chaintracks/Api/ChaintracksClientApi' import { ChaintracksServiceClient } from './chaintracker/chaintracks/ChaintracksServiceClient' -import { tstnArcadeUrl, tstnChaintracksUrl } from './networkConfig' +import { GoChaintracksServiceClient } from './chaintracker/chaintracks/GoChaintracksServiceClient' +import { publicArcadeUrl, stnArcadeUrl, stnChaintracksUrl, tstnArcadeUrl, tstnChaintracksUrl } from './networkConfig' -export function createDefaultWalletServicesOptions ( +function stripTrailingSlash(value: string): string { + let end = value.length + while (end > 0 && value[end - 1] === '/') end-- + return value.slice(0, end) +} + +function configuredChaintracksClient(chain: Chain, serviceUrl: string): ChaintracksClientApi { + let path = '' + try { + path = stripTrailingSlash(new URL(serviceUrl).pathname) + } catch { + // Preserve the legacy client's existing validation/error behavior for an + // operator-supplied non-URL value. + } + if (path.endsWith('/v2')) return new GoChaintracksServiceClient(chain, serviceUrl) + return new ChaintracksServiceClient(chain, serviceUrl) +} + +/** + * Returns the credential-free default ChainTracks client for a supported + * public network, or an operator-configured client for stn/tstn. + */ +export function createDefaultChaintracksClient(chain: Exclude): ChaintracksClientApi { + switch (chain) { + case 'main': + case 'test': + case 'ttn': + return new GoChaintracksServiceClient(chain, arcadeDefaultUrl(chain)!, { + apiPrefix: '/chaintracks/v2' + }) + case 'stn': + return configuredChaintracksClient(chain, stnChaintracksUrl()) + case 'tstn': + return configuredChaintracksClient(chain, tstnChaintracksUrl()) + } +} + +export function createDefaultWalletServicesOptions( ...[ chain, arcCallbackUrl, @@ -43,25 +81,16 @@ export function createDefaultWalletServicesOptions ( ] ): WalletServicesOptions { if (chain === 'mock') { - throw new Error('createDefaultWalletServicesOptions does not support \'mock\' chain. Use MockServices directly.') + throw new Error("createDefaultWalletServicesOptions does not support 'mock' chain. Use MockServices directly.") } deploymentId ||= `wallet-toolbox-${randomBytesHex(16)}` - // const chaintracksUrl = `https://npm-registry.babbage.systems:${chain === 'main' ? 8084 : 8083}` - let chaintracksUrl: string - if (chain === 'ttn') { - chaintracksUrl = 'https://arcade-v2-ttn-us-1.bsvblockchain.tech/chaintracks/v1' - } else if (chain === 'tstn') { - chaintracksUrl = tstnChaintracksUrl() - } else { - chaintracksUrl = `https://${chain}net-chaintracks.babbage.systems` - } // The mainnet endpoint is always used since these are fiat exchange rates, // independent of the chain being used. const chaintracksFiatExchangeRatesUrl = 'https://mainnet-chaintracks.babbage.systems/getFiatExchangeRates' - chaintracks ||= new ChaintracksServiceClient(chain, chaintracksUrl) + chaintracks ||= createDefaultChaintracksClient(chain) const o: WalletServicesOptions = { chain, @@ -130,31 +159,32 @@ export function createDefaultWalletServicesOptions ( /** * Default Arcade (bsv-blockchain/arcade) endpoint per chain. - * Returns undefined when no public default is known for the chain (e.g. testnet not yet deployed). + * Returns undefined when no public default is known for the chain. */ -export function arcadeDefaultUrl (chain: Chain): string | undefined { +export function arcadeDefaultUrl(chain: Chain): string | undefined { switch (chain) { case 'main': - return 'https://arcade-v2-us-1.bsvblockchain.tech' + case 'test': case 'ttn': - return 'https://arcade-v2-ttn-us-1.bsvblockchain.tech' + return publicArcadeUrl(chain) + case 'stn': + return stnArcadeUrl() case 'tstn': // Private per-deployment endpoint supplied via TSTN_ARCADE_URL (undefined when unset). return tstnArcadeUrl() - case 'test': - // No public testnet Arcade endpoint deployed yet. - return undefined case 'mock': return undefined } } -export function arcDefaultUrl (chain: Chain): string { +export function arcDefaultUrl(chain: Chain): string { switch (chain) { case 'main': return 'https://arc.taal.com' case 'test': return 'https://arc-test.taal.com' + case 'stn': + return stnArcadeUrl() ?? '' case 'ttn': return 'https://arcade-v2-ttn-us-1.bsvblockchain.tech/' case 'tstn': @@ -165,6 +195,6 @@ export function arcDefaultUrl (chain: Chain): string { } } -export function arcGorillaPoolUrl (chain: Chain): string | undefined { +export function arcGorillaPoolUrl(chain: Chain): string | undefined { return chain === 'main' ? 'https://arc.gorillapool.io' : undefined } diff --git a/packages/wallet/wallet-toolbox/src/services/networkConfig.ts b/packages/wallet/wallet-toolbox/src/services/networkConfig.ts index de7774641..760606f07 100644 --- a/packages/wallet/wallet-toolbox/src/services/networkConfig.ts +++ b/packages/wallet/wallet-toolbox/src/services/networkConfig.ts @@ -1,31 +1,52 @@ +import { Chain } from '../sdk/types' + /** - * Runtime service-endpoint configuration for the `tstn` (Teranode Scaling Test Net) network. + * Runtime service-endpoint configuration for Teranode networks that do not + * have a public, operator-independent service endpoint. * - * Unlike `main`, `test`, and `ttn`, the tstn service endpoints are not public and must not be + * Unlike `main`, `test`, and `ttn`, the stn/tstn service endpoints are not public and must not be * hardcoded in this (public) source tree. They are supplied at runtime through environment * variables: * + * STN_ARCADE_URL STN Arcade broadcaster / ARC endpoint base. + * STN_CHAINTRACKS_URL STN ChainTracks service URL. * TSTN_ARCADE_URL Arcade broadcaster / ARC endpoint base. Also the fallback host for * ChainTracks when TSTN_CHAINTRACKS_URL is unset * (`${TSTN_ARCADE_URL}/chaintracks/v1`, mirroring the ttn layout). * TSTN_CHAINTRACKS_URL ChainTracks service URL. * - * tstn runs only Arcade (broadcast + merkle proofs) and ChainTracks (headers); there is no - * WhatsOnChain / block-explorer service for tstn, so no WhatsOnChain endpoint is configured and + * stn/tstn run only operator-configured Arcade and ChainTracks services; there is no + * documented WhatsOnChain service for them, so no WhatsOnChain endpoint is configured and * the WhatsOnChain-only lookups (raw tx, utxo status, txid status, script-hash history) are not - * available on tstn. + * available on stn/tstn. * - * `process` is accessed defensively so importing this module remains safe in browser bundles; - * tstn is a server-side network and these variables are only read when the selected chain is - * tstn. + * `process` is accessed defensively so importing this module remains safe in + * browser bundles. Browser applications can still supply an explicit + * ChaintracksClientApi without relying on environment variables. */ -function readEnv (name: string): string | undefined { +function readEnv(name: string): string | undefined { const env = typeof process !== 'undefined' ? process.env : undefined const value = env?.[name] return value != null && value.trim() !== '' ? value.trim() : undefined } +/** Credential-free public Arcade host for supported networks. */ +export function publicArcadeUrl(chain: Chain): string | undefined { + switch (chain) { + case 'main': + return 'https://arcade-v2-us-1.bsvblockchain.tech' + case 'test': + return 'https://arcade-v2-testnet-us-1.bsvblockchain.tech' + case 'ttn': + return 'https://arcade-v2-ttn-us-1.bsvblockchain.tech' + case 'stn': + case 'tstn': + case 'mock': + return undefined + } +} + const stripTrailingSlash = (url: string): string => { let end = url.length while (end > 0 && url[end - 1] === '/') end-- @@ -33,15 +54,20 @@ const stripTrailingSlash = (url: string): string => { } /** Arcade broadcaster / ARC endpoint for tstn, or `undefined` when `TSTN_ARCADE_URL` is unset. */ -export function tstnArcadeUrl (): string | undefined { +export function tstnArcadeUrl(): string | undefined { return readEnv('TSTN_ARCADE_URL') } +/** Arcade broadcaster / ARC endpoint for stn, or `undefined` when unset. */ +export function stnArcadeUrl(): string | undefined { + return readEnv('STN_ARCADE_URL') +} + /** * ChainTracks service URL for tstn. Falls back to `${TSTN_ARCADE_URL}/chaintracks/v1` when * `TSTN_CHAINTRACKS_URL` is unset (mirrors the ttn layout). Throws when neither is configured. */ -export function tstnChaintracksUrl (): string { +export function tstnChaintracksUrl(): string { const explicit = readEnv('TSTN_CHAINTRACKS_URL') if (explicit != null) return explicit const arcade = tstnArcadeUrl() @@ -50,3 +76,17 @@ export function tstnChaintracksUrl (): string { 'tstn chain requires a ChainTracks URL: set TSTN_CHAINTRACKS_URL (or TSTN_ARCADE_URL) in the environment.' ) } + +/** + * ChainTracks service URL for stn. Falls back to the configured Arcade host's + * legacy-compatible path when STN_CHAINTRACKS_URL is unset. + */ +export function stnChaintracksUrl(): string { + const explicit = readEnv('STN_CHAINTRACKS_URL') + if (explicit != null) return explicit + const arcade = stnArcadeUrl() + if (arcade != null) return `${stripTrailingSlash(arcade)}/chaintracks/v1` + throw new Error( + 'stn chain requires a ChainTracks URL: set STN_CHAINTRACKS_URL (or STN_ARCADE_URL) in the environment.' + ) +} diff --git a/packages/wallet/wallet-toolbox/src/services/providers/WhatsOnChain.ts b/packages/wallet/wallet-toolbox/src/services/providers/WhatsOnChain.ts index b16ba53e7..e51ac9aae 100644 --- a/packages/wallet/wallet-toolbox/src/services/providers/WhatsOnChain.ts +++ b/packages/wallet/wallet-toolbox/src/services/providers/WhatsOnChain.ts @@ -1,4 +1,4 @@ -import { Beef, HexString, Utils, WhatsOnChainConfig } from '@bsv/sdk' +import { Beef, HexString, HttpClientRequestOptions, HttpClientResponse, Utils, WhatsOnChainConfig } from '@bsv/sdk' import { convertProofToMerklePath } from '../../utility/tscProofToMerklePath' import SdkWhatsOnChain from './SdkWhatsOnChain' import { Chain } from '../../sdk/types' @@ -31,10 +31,38 @@ import { ScriptHashHistoryResponse } from './whatsOnChainHelpers' +export interface WalletToolboxWhatsOnChainConfig extends WhatsOnChainConfig { + /** Optional request-start gate used by ChainTracks' shared public-rate scheduler. */ + requestGate?: () => Promise +} + export class WhatsOnChainNoServices extends SdkWhatsOnChain { - constructor(chain: Chain = 'main', config: WhatsOnChainConfig = {}) { + private readonly requestGate?: () => Promise + + constructor(chain: Chain = 'main', config: WalletToolboxWhatsOnChainConfig = {}) { if (chain === 'mock') throw new Error("WhatsOnChain does not support 'mock' chain. Use MockServices directly.") super(chain, config) + this.requestGate = config.requestGate + } + + private async requestWithAnonymousAuthFallback( + url: string, + requestOptions: HttpClientRequestOptions + ): Promise> { + await this.requestGate?.() + const response = await this.httpClient.request(url, requestOptions) + if ((response.status !== 401 && response.status !== 403) || this.apiKey.trim() === '') { + return response + } + + // Treat the anonymous retry as another public request start so a stale + // key cannot create a burst above the documented keyless allowance. + if (this.requestGate != null) await this.requestGate() + else await wait(350) + return await this.httpClient.request(url, { + method: 'GET', + headers: { Accept: 'application/json' } + }) } /** @@ -492,7 +520,7 @@ export class WhatsOnChainNoServices extends SdkWhatsOnChain { const url = `${this.URL}/block/${hash}/header` for (let retry = 0; retry < 2; retry++) { - const response = await this.httpClient.request(url, requestOptions) + const response = await this.requestWithAnonymousAuthFallback(url, requestOptions) if (response.statusText === 'Too Many Requests' && retry < 2) { await wait(2000) continue @@ -522,7 +550,7 @@ export class WhatsOnChainNoServices extends SdkWhatsOnChain { const url = `${this.URL}/chain/info` for (let retry = 0; retry < 2; retry++) { - const response = await this.httpClient.request(url, requestOptions) + const response = await this.requestWithAnonymousAuthFallback(url, requestOptions) if (response.statusText === 'Too Many Requests' && retry < 2) { await wait(2000) continue @@ -545,7 +573,7 @@ export class WhatsOnChainNoServices extends SdkWhatsOnChain { export class WhatsOnChain extends WhatsOnChainNoServices { services: Services - constructor(chain: Chain = 'main', config: WhatsOnChainConfig = {}, services?: Services) { + constructor(chain: Chain = 'main', config: WalletToolboxWhatsOnChainConfig = {}, services?: Services) { super(chain, config) this.services = services || new Services(chain) } diff --git a/packages/wallet/wallet-toolbox/src/services/providers/__tests/WhatsOnChain.keyless.test.ts b/packages/wallet/wallet-toolbox/src/services/providers/__tests/WhatsOnChain.keyless.test.ts new file mode 100644 index 000000000..b661206d6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/services/providers/__tests/WhatsOnChain.keyless.test.ts @@ -0,0 +1,96 @@ +import { HttpClient, HttpClientRequestOptions, HttpClientResponse } from '@bsv/sdk' +import { WhatsOnChain, WhatsOnChainNoServices, WocChainInfo } from '../WhatsOnChain' + +describe('WhatsOnChain optional authentication', () => { + afterEach(() => { + jest.useRealTimers() + }) + + test('retries anonymously when a configured key is rejected', async () => { + const requests: HttpClientRequestOptions[] = [] + const value: WocChainInfo = { + chain: 'main', + blocks: 100, + headers: 100, + bestblockhash: '00'.repeat(32), + difficulty: 1, + mediantime: 1, + verificationprogress: 1, + pruned: false, + chainwork: '00'.repeat(32) + } + let call = 0 + const httpClient: HttpClient = { + async request(_url: string, options: HttpClientRequestOptions): Promise> { + requests.push(options) + call++ + if (call === 1) { + return { ok: false, status: 401, statusText: 'Unauthorized', data: {} as T } + } + return { ok: true, status: 200, statusText: 'OK', data: value as T } + } + } + const requestGate = jest.fn(async () => {}) + const woc = new WhatsOnChainNoServices('main', { + apiKey: 'rejected-key', + httpClient, + requestGate + }) + + await expect(woc.getChainInfo()).resolves.toEqual(value) + expect(requests).toHaveLength(2) + expect(requests[0].headers).toMatchObject({ Authorization: 'rejected-key' }) + expect(requests[1].headers).not.toHaveProperty('Authorization') + expect(requestGate).toHaveBeenCalledTimes(2) + }) + + test('uses keyless requests directly and returns undefined for an unknown block', async () => { + const httpClient: HttpClient = { + async request(): Promise> { + return { ok: false, status: 404, statusText: 'Not Found', data: undefined as T } + } + } + const requestGate = jest.fn(async () => {}) + const woc = new WhatsOnChainNoServices('main', { httpClient, requestGate }) + + await expect(woc.getBlockHeaderByHash('00'.repeat(32))).resolves.toBeUndefined() + expect(requestGate).toHaveBeenCalledTimes(1) + }) + + test('serializes anonymous auth fallback and rate-limit retries without a request gate', async () => { + jest.useFakeTimers() + const value: WocChainInfo = { + chain: 'main', + blocks: 100, + headers: 100, + bestblockhash: '00'.repeat(32), + difficulty: 1, + mediantime: 1, + verificationprogress: 1, + pruned: false, + chainwork: '00'.repeat(32) + } + const responses: Array> = [ + { ok: false, status: 403, statusText: 'Forbidden', data: {} as WocChainInfo }, + { ok: false, status: 429, statusText: 'Too Many Requests', data: {} as WocChainInfo }, + { ok: true, status: 200, statusText: 'OK', data: value } + ] + const httpClient: HttpClient = { + async request(): Promise> { + return responses.shift() as HttpClientResponse + } + } + const woc = new WhatsOnChainNoServices('main', { apiKey: 'stale-key', httpClient }) + const result = woc.getChainInfo() + + await jest.advanceTimersByTimeAsync(350) + await jest.advanceTimersByTimeAsync(2000) + await expect(result).resolves.toEqual(value) + }) + + test('rejects mock construction and supports an injected Services instance', () => { + expect(() => new WhatsOnChainNoServices('mock')).toThrow("does not support 'mock' chain") + const services = {} as any + expect(new WhatsOnChain('main', {}, services).services).toBe(services) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/index.all.ts b/packages/wallet/wallet-toolbox/src/storage/index.all.ts index 3cc82893a..f9897e7b1 100644 --- a/packages/wallet/wallet-toolbox/src/storage/index.all.ts +++ b/packages/wallet/wallet-toolbox/src/storage/index.all.ts @@ -4,6 +4,7 @@ export * from './StorageSyncReader' export * from './remoting/StorageClient' export * from './remoting/StorageServer' export * from './remoting/KnexSessionManager' +export * from './remoting/KnexPaymentReplayStore' export * from './schema/KnexMigrations' export * from './StorageKnex' export * from './StorageIdb' diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/KnexPaymentReplayStore.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/KnexPaymentReplayStore.ts new file mode 100644 index 000000000..f93d8b470 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/KnexPaymentReplayStore.ts @@ -0,0 +1,46 @@ +import type { PaymentReplayStore } from '@bsv/payment-express-middleware' +import type { Knex } from 'knex' + +export const PAYMENT_REPLAY_TABLE = 'payment_replays' + +function isDuplicate(error: unknown): boolean { + if (error == null || typeof error !== 'object') return false + const value = error as { code?: unknown; errno?: unknown } + return ( + value.code === 'ER_DUP_ENTRY' || + value.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || + value.code === 'SQLITE_CONSTRAINT_UNIQUE' || + value.errno === 1062 + ) +} + +/** Durable, replica-safe BRC-105 transaction replay claims. */ +export class KnexPaymentReplayStore implements PaymentReplayStore { + constructor( + private readonly knex: Knex, + private readonly ttlDays: number = 365 + ) { + if (!Number.isSafeInteger(ttlDays) || (ttlDays !== -1 && ttlDays < 1)) { + throw new TypeError('KnexPaymentReplayStore ttlDays must be -1 or a positive integer.') + } + } + + async claim(transactionId: string): Promise { + const now = new Date() + try { + await this.knex(PAYMENT_REPLAY_TABLE).insert({ + transactionId, + createdAt: now, + expiresAt: this.ttlDays === -1 ? null : new Date(now.getTime() + this.ttlDays * 24 * 60 * 60 * 1_000) + }) + return true + } catch (error) { + if (isDuplicate(error)) return false + throw error + } + } + + async pruneExpired(now = new Date()): Promise { + return await this.knex(PAYMENT_REPLAY_TABLE).whereNotNull('expiresAt').where('expiresAt', '<=', now).delete() + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts index 3f6f9f161..9c39321f0 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts @@ -16,6 +16,7 @@ import { import express, { Request, Response } from 'express' import { AuthMiddlewareOptions, AuthRequest, createAuthMiddleware } from '@bsv/auth-express-middleware' import { createPaymentMiddleware } from '@bsv/payment-express-middleware' +import type { PaymentReplayStore } from '@bsv/payment-express-middleware' import { Options as RateLimitOptions, rateLimit } from 'express-rate-limit' import { Wallet } from '../../Wallet' import { StorageProvider } from '../StorageProvider' @@ -39,7 +40,11 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readBodyLimitBytes, + readResourceLimit, + readResourceProfile, securityHeaders, type HttpServerPolicyDefaults, type SecurityHeadersOptions @@ -125,6 +130,20 @@ const actionBatchRpcMethods = new Set([ 'renewActionBatch' ]) +const topLevelLimitArgument = new Map([ + ['listActions', 1], + ['listCertificates', 1], + ['listOutputs', 1] +]) + +const pagedLimitArgument = new Map([ + ['findCertificatesAuth', 1], + ['findOutputBaskets', 1], + ['findOutputBasketsAuth', 1], + ['findOutputsAuth', 1], + ['findProvenTxReqs', 0] +]) + interface RpcDispatchResult { found: boolean result?: unknown @@ -175,7 +194,7 @@ export interface WalletStorageServerOptions { trustProxy?: TrustProxySetting /** Exact browser origins allowed to call this server. Omit for public CORS. */ allowedOrigins?: string[] - /** Per-process in-flight request ceiling. Defaults to 200. */ + /** Per-process in-flight request ceiling. Defaults to the selected resource profile (24 in standard). */ maxConcurrentRequests?: number /** Node HTTP timeout/connection policy overrides. */ http?: Partial @@ -190,6 +209,16 @@ export interface WalletStorageServerOptions { * authorization, storage dispatch, and response formulation. */ telemetry?: TelemetryConfig + /** Default item limit inserted for list/find RPC calls that omit one. Default: 1,000. */ + defaultRpcListLimit?: number + /** Largest caller-selected list/find item limit. Use -1 to disable this operator ceiling. */ + maxRpcListLimit?: number + /** Maximum elements in any decoded request array. Use -1 only for trusted callers. */ + maxRpcArrayItems?: number + /** Maximum serialized JSON-RPC response bytes. Use -1 to disable. */ + maxRpcResponseBytes?: number + /** Durable BRC-105 replay claims for monetized multi-replica deployments. */ + paymentReplayStore?: PaymentReplayStore } export class StorageServer { @@ -212,6 +241,11 @@ export class StorageServer { private readonly logRpcRequests: boolean private readonly telemetry: Telemetry private readonly telemetryConfig?: TelemetryConfig + private readonly defaultRpcListLimit: number + private readonly maxRpcListLimit: number + private readonly maxRpcArrayItems: number + private readonly maxRpcResponseBytes: number + private readonly paymentReplayStore?: PaymentReplayStore constructor(storage: StorageProvider, options: WalletStorageServerOptions) { this.storage = storage @@ -226,7 +260,14 @@ export class StorageServer { this.preAuthRateLimitOptions = options.preAuthRateLimit this.trustProxy = options.trustProxy this.allowedOrigins = options.allowedOrigins - this.maxConcurrentRequests = options.maxConcurrentRequests ?? 200 + const profile = readResourceProfile('WALLET_STORAGE') + this.maxConcurrentRequests = + options.maxConcurrentRequests ?? + readResourceLimit( + 'WALLET_STORAGE', + 'MAX_CONCURRENT_REQUESTS', + profileValue(profile, { small: 8, standard: 24, highThroughput: 96 }) + ) this.httpPolicy = { requestTimeoutMs: 2 * 60 * 1000, headersTimeoutMs: 15_000, @@ -239,6 +280,46 @@ export class StorageServer { this.logRpcRequests = options.logRpcRequests ?? true this.telemetryConfig = options.telemetry this.telemetry = new Telemetry(options.telemetry) + this.paymentReplayStore = options.paymentReplayStore + this.defaultRpcListLimit = + options.defaultRpcListLimit ?? + readResourceLimit( + 'WALLET_STORAGE', + 'RPC_DEFAULT_LIST_LIMIT', + profileValue(profile, { small: 500, standard: 1_000, highThroughput: 1_000 }) + ) + this.maxRpcListLimit = + options.maxRpcListLimit ?? + readResourceLimit( + 'WALLET_STORAGE', + 'RPC_MAX_LIST_LIMIT', + profileValue(profile, { small: 500, standard: 1_000, highThroughput: 5_000 }) + ) + this.maxRpcArrayItems = + options.maxRpcArrayItems ?? + readResourceLimit( + 'WALLET_STORAGE', + 'RPC_MAX_ARRAY_ITEMS', + profileValue(profile, { small: 250_000, standard: 1_000_000, highThroughput: 4_000_000 }) + ) + this.maxRpcResponseBytes = + options.maxRpcResponseBytes ?? + readResourceLimit( + 'WALLET_STORAGE', + 'RPC_MAX_RESPONSE_BYTES', + profileValue(profile, { + small: 4 * 1024 * 1024, + standard: 8 * 1024 * 1024, + highThroughput: 32 * 1024 * 1024 + }) + ) + if ( + this.defaultRpcListLimit !== -1 && + this.maxRpcListLimit !== -1 && + this.defaultRpcListLimit > this.maxRpcListLimit + ) { + throw new RangeError('defaultRpcListLimit must not exceed maxRpcListLimit') + } const legacyLogShortReqs = (options as unknown as Record)['logShortReqs'] if (legacyLogShortReqs) { @@ -279,6 +360,7 @@ export class StorageServer { private setupRoutes(): void { configureTrustProxy(this.app, this.trustProxy) this.app.disable('x-powered-by') + this.app.use(initialDoubleSlashCompatibility) this.app.use( securityHeaders({ environmentPrefix: 'WALLET_STORAGE', @@ -315,7 +397,14 @@ export class StorageServer { this.app.set('json escape', true) this.app.use( express.json({ - limit: readBodyLimitBytes('WALLET_STORAGE_JSON', 30 * 1024 * 1024) + limit: readBodyLimitBytes( + 'WALLET_STORAGE_JSON', + profileValue(readResourceProfile('WALLET_STORAGE'), { + small: 2 * 1024 * 1024, + standard: 8 * 1024 * 1024, + highThroughput: 32 * 1024 * 1024 + }) + ) }) ) // Authentication must see the exact binary body bytes, so parse octet @@ -333,6 +422,11 @@ export class StorageServer { res.send('User-agent: *\nDisallow: /') }) + this.app.get('/healthz', (_req: Request, res: Response) => { + res.setHeader('Cache-Control', 'no-store') + res.status(200).json({ status: 'ok' }) + }) + this.app.get('/', (req: Request, res: Response) => { res.type('text/plain') res.send(`BRC-100 ${this.wallet.chain}Net Storage Provider.`) @@ -340,6 +434,7 @@ export class StorageServer { const options: AuthMiddlewareOptions = { wallet: this.wallet as WalletInterface, + transportLimits: { maxResponseBytes: this.maxRpcResponseBytes }, ...(this.telemetryConfig == null ? {} : { telemetry: this.telemetryConfig }) } if (this.sessionManager != null) options.sessionManager = this.sessionManager @@ -358,7 +453,8 @@ export class StorageServer { this.app.use( createPaymentMiddleware({ wallet: this.wallet, - calculateRequestPrice: this.calculateRequestPrice || (() => 100) + calculateRequestPrice: this.calculateRequestPrice || (() => 100), + replayStore: this.paymentReplayStore }) ) } @@ -439,12 +535,13 @@ export class StorageServer { const { jsonrpc, method, id } = req.body const params = (requestUsesBinary ? decodeBinaryJsonValue(req.body.params) : req.body.params) as any[] - if (jsonrpc !== '2.0' || !method || typeof method !== 'string') { + if (jsonrpc !== '2.0' || !method || typeof method !== 'string' || !Array.isArray(params)) { return this.sendRpc(res, useBinary, { error: { code: -32600, message: 'Invalid Request' } }, 400) } const logObj = this.createRpcLog(req, method, id, params) try { + this.enforceRpcRequestBudgets(method, params) const dispatch = await this.dispatchRpcCall(method, params, req, logObj, rpcSpan) if (!dispatch.found) { return this.sendRpc( @@ -499,9 +596,97 @@ export class StorageServer { private sendRpc(res: Response, useBinary: boolean, payload: unknown, status: number = 200): Response { res.set('X-Content-Type-Options', 'nosniff') + const serialized = stringifyJsonRpc(payload, useBinary) + if (this.maxRpcResponseBytes !== -1 && Buffer.byteLength(serialized, 'utf8') > this.maxRpcResponseBytes) { + return res.status(413).json({ + jsonrpc: '2.0', + error: { + code: -32005, + message: 'The requested response exceeds the configured service limit.' + }, + id: (payload as { id?: unknown } | null)?.id + }) + } // Normalize with the negotiated binary replacer, then let Express emit // the JSON response through its escaping-aware JSON sink. - return res.status(status).json(JSON.parse(stringifyJsonRpc(payload, useBinary))) + return res.status(status).json(JSON.parse(serialized)) + } + + private enforceRpcRequestBudgets(method: string, params: any[]): void { + this.enforceRpcArrayBudget(params) + + const topLevelIndex = topLevelLimitArgument.get(method) + if (topLevelIndex != null) { + const args = this.objectArgument(params, topLevelIndex) + args.limit = this.normalizedRpcLimit(args.limit) + } + const pagedIndex = pagedLimitArgument.get(method) + if (pagedIndex != null) { + const args = this.objectArgument(params, pagedIndex) + const paged = args.paged ?? {} + if (typeof paged !== 'object' || Array.isArray(paged)) { + throw new TypeError('paged must be an object') + } + paged.limit = this.normalizedRpcLimit(paged.limit) + args.paged = paged + } + if (method === 'getSyncChunk') { + const args = this.objectArgument(params, 0) + args.maxItems = this.normalizedRpcLimit(args.maxItems) + if ( + this.maxRpcResponseBytes !== -1 && + (!Number.isSafeInteger(args.maxRoughSize) || args.maxRoughSize > this.maxRpcResponseBytes) + ) { + args.maxRoughSize = this.maxRpcResponseBytes + } + } + } + + private enforceRpcArrayBudget(params: any[]): void { + if (this.maxRpcArrayItems === -1) return + + const pending: Array<{ value: unknown; depth: number }> = [{ value: params, depth: 0 }] + const seen = new Set() + while (pending.length > 0) { + const current = pending.pop()! + if (current.depth > 64) throw new RangeError('RPC parameter nesting exceeds 64 levels') + if (current.value == null || typeof current.value !== 'object') continue + if (seen.has(current.value)) continue + seen.add(current.value) + if (Array.isArray(current.value) && current.value.length > this.maxRpcArrayItems) { + throw new RangeError(`RPC arrays must not exceed ${this.maxRpcArrayItems} items`) + } + for (const value of Object.values(current.value)) { + pending.push({ value, depth: current.depth + 1 }) + } + } + } + + private objectArgument(params: any[], index: number): Record { + const value = params[index] + if (value == null) { + const created: Record = {} + params[index] = created + return created + } + if (typeof value !== 'object' || Array.isArray(value)) { + throw new TypeError(`RPC parameter ${index} must be an object`) + } + return value + } + + private normalizedRpcLimit(value: unknown): number { + if (value == null) { + return this.defaultRpcListLimit === -1 ? Number.MAX_SAFE_INTEGER : this.defaultRpcListLimit + } + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new RangeError('RPC list limits must be positive safe integers') + } + const limit = Number(value) + if (this.maxRpcListLimit !== -1 && limit > this.maxRpcListLimit) { + throw new RangeError(`RPC list limits must not exceed ${this.maxRpcListLimit}`) + } + return limit } private sendRpcError(res: Response, useBinary: boolean, id: unknown, error: unknown): Response { diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexPaymentReplayStore.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexPaymentReplayStore.test.ts new file mode 100644 index 000000000..b119309a0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/KnexPaymentReplayStore.test.ts @@ -0,0 +1,83 @@ +import { knex as makeKnex, type Knex } from 'knex' +import { KnexMigrations, PAYMENT_REPLAY_MIGRATION } from '../../schema/KnexMigrations' +import { KnexPaymentReplayStore } from '../KnexPaymentReplayStore' + +describe('KnexPaymentReplayStore', () => { + let database: Knex + + beforeEach(async () => { + database = makeKnex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + const migrations = new KnexMigrations('test', 'payment replay tests', '1'.repeat(64), 1024) + await (await migrations.getMigration(PAYMENT_REPLAY_MIGRATION)).up(database) + }) + + afterEach(async () => { + await database.destroy() + }) + + it('atomically accepts a transaction once and prunes expired claims', async () => { + const store = new KnexPaymentReplayStore(database, 1) + await expect(store.claim('transaction-id')).resolves.toBe(true) + await expect(store.claim('transaction-id')).resolves.toBe(false) + await database('payment_replays').update({ expiresAt: new Date(0) }) + await expect(store.pruneExpired()).resolves.toBe(1) + }) + + it('supports explicit non-expiring claims and rejects invalid TTL values', () => { + expect(() => new KnexPaymentReplayStore(database, -1)).not.toThrow() + expect(() => new KnexPaymentReplayStore(database, 0)).toThrow('ttlDays') + expect(() => new KnexPaymentReplayStore(database, -2)).toThrow('ttlDays') + expect(() => new KnexPaymentReplayStore(database, 1.5)).toThrow('ttlDays') + }) + + it('stores non-expiring claims without an expiry timestamp', async () => { + const store = new KnexPaymentReplayStore(database, -1) + + await expect(store.claim('non-expiring')).resolves.toBe(true) + await expect(database('payment_replays').where({ transactionId: 'non-expiring' }).first()).resolves.toMatchObject({ + expiresAt: null, + transactionId: 'non-expiring' + }) + }) + + it.each([ + { code: 'ER_DUP_ENTRY' }, + { code: 'SQLITE_CONSTRAINT_PRIMARYKEY' }, + { code: 'SQLITE_CONSTRAINT_UNIQUE' }, + { errno: 1062 } + ])('recognizes supported duplicate-key errors without accepting a replay', async duplicateError => { + const insert = jest.fn(async () => await Promise.reject(duplicateError)) + const fakeKnex = jest.fn(() => ({ insert })) as unknown as Knex + const store = new KnexPaymentReplayStore(fakeKnex) + + await expect(store.claim('duplicate')).resolves.toBe(false) + }) + + it.each([null, 'database failed', { code: 'SOME_OTHER_ERROR' }])( + 'does not hide non-duplicate database failures', + async databaseError => { + const insert = jest.fn(async () => await Promise.reject(databaseError)) + const fakeKnex = jest.fn(() => ({ insert })) as unknown as Knex + const store = new KnexPaymentReplayStore(fakeKnex) + + await expect(store.claim('failed')).rejects.toBe(databaseError) + } + ) + + it('passes the requested cutoff to the pruning query', async () => { + const remove = jest.fn(async () => 3) + const where = jest.fn(() => ({ delete: remove })) + const whereNotNull = jest.fn(() => ({ where })) + const fakeKnex = jest.fn(() => ({ whereNotNull })) as unknown as Knex + const store = new KnexPaymentReplayStore(fakeKnex) + const cutoff = new Date('2026-08-04T00:00:00.000Z') + + await expect(store.pruneExpired(cutoff)).resolves.toBe(3) + expect(whereNotNull).toHaveBeenCalledWith('expiresAt') + expect(where).toHaveBeenCalledWith('expiresAt', '<=', cutoff) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts index addd1dd34..7100d43c4 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts @@ -328,6 +328,171 @@ describe('StorageServer JSON-RPC boundary', () => { }) }) + test('defaults and bounds direct RPC list limits before storage dispatch', async () => { + const server = makeServer( + {}, + { + defaultRpcListLimit: 100, + maxRpcListLimit: 1_000 + } + ) + const listParams: any[] = [{ identityKey: 'alice' }, {}] + await invoke(server, 'enforceRpcRequestBudgets', 'listActions', listParams) + expect(listParams[1].limit).toBe(100) + + const findParams: any[] = [{ identityKey: 'alice' }, { partial: {} }] + await invoke(server, 'enforceRpcRequestBudgets', 'findOutputsAuth', findParams) + expect(findParams[1].paged).toEqual({ limit: 100 }) + + await expect( + invoke(server, 'enforceRpcRequestBudgets', 'listOutputs', [{ identityKey: 'alice' }, { limit: 1_001 }]) + ).rejects.toThrow('must not exceed 1000') + }) + + test('validates configured, paged, and synchronization RPC limits', async () => { + expect(() => makeServer({}, { defaultRpcListLimit: 11, maxRpcListLimit: 10 })).toThrow( + 'defaultRpcListLimit must not exceed maxRpcListLimit' + ) + + const server = makeServer( + {}, + { + defaultRpcListLimit: 5, + maxRpcListLimit: 10, + maxRpcResponseBytes: 128 + } + ) + await expect(invoke(server, 'enforceRpcRequestBudgets', 'findOutputsAuth', [{}, { paged: [] }])).rejects.toThrow( + 'paged must be an object' + ) + await expect(invoke(server, 'enforceRpcRequestBudgets', 'listActions', [{}, []])).rejects.toThrow( + 'RPC parameter 1 must be an object' + ) + await expect(invoke(server, 'enforceRpcRequestBudgets', 'listActions', [{}, { limit: 0 }])).rejects.toThrow( + 'positive safe integers' + ) + await expect( + invoke(server, 'enforceRpcRequestBudgets', 'listActions', [{}, { limit: Number.MAX_SAFE_INTEGER + 1 }]) + ).rejects.toThrow('positive safe integers') + + const syncParams: any[] = [{ maxRoughSize: 'unbounded' }] + await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', syncParams) + expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128 }) + + const oversizedSyncParams: any[] = [{ maxItems: 4, maxRoughSize: 129 }] + await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', oversizedSyncParams) + expect(oversizedSyncParams[0]).toEqual({ maxItems: 4, maxRoughSize: 128 }) + + const unlimited = makeServer( + {}, + { + defaultRpcListLimit: -1, + maxRpcArrayItems: -1, + maxRpcListLimit: -1, + maxRpcResponseBytes: -1 + } + ) + const unlimitedParams: any[] = [null] + await invoke(unlimited, 'enforceRpcRequestBudgets', 'getSyncChunk', unlimitedParams) + expect(unlimitedParams[0]).toEqual({ maxItems: Number.MAX_SAFE_INTEGER }) + await expect( + invoke(unlimited, 'enforceRpcRequestBudgets', 'getSettings', [Array.from({ length: 10_000 }, () => 1)]) + ).resolves.toBeUndefined() + }) + + test('serves public service metadata without storage access', () => { + const server = makeServer() + const app = Reflect.get(server, 'app') + const healthLayer = app.router.stack.find((layer: any) => layer.route?.path === '/healthz') + const healthHandler = healthLayer.route.stack[0].handle + const response = { + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn(), + type: jest.fn(), + send: jest.fn() + } + response.status.mockReturnValue(response) + response.json.mockReturnValue(response) + response.type.mockReturnValue(response) + response.send.mockReturnValue(response) + + healthHandler({} as Request, response as unknown as Response) + + expect(response.setHeader).toHaveBeenCalledWith('Cache-Control', 'no-store') + expect(response.status).toHaveBeenCalledWith(200) + expect(response.json).toHaveBeenCalledWith({ status: 'ok' }) + + for (const [path, expected] of [ + ['/robots.txt', 'User-agent: *\nDisallow: /'], + ['/', 'BRC-100 testNet Storage Provider.'] + ]) { + const routeLayer = app.router.stack.find((layer: any) => layer.route?.path === path) + routeLayer.route.stack[0].handle({} as Request, response as unknown as Response) + expect(response.send).toHaveBeenLastCalledWith(expected) + } + expect(response.type).toHaveBeenCalledTimes(2) + expect(response.type).toHaveBeenCalledWith('text/plain') + }) + + test('supports default and operator-defined request pricing', () => { + const paymentWallet = { + chain: 'test', + internalizeAction: jest.fn() + } as any + expect(() => makeServer({}, { monetize: true, wallet: paymentWallet })).not.toThrow() + expect(() => + makeServer( + {}, + { + monetize: true, + wallet: paymentWallet, + calculateRequestPrice: () => 42 + } + ) + ).not.toThrow() + }) + + test('handles cyclic RPC values while rejecting excessive array size and nesting', async () => { + const server = makeServer({}, { maxRpcArrayItems: 2 }) + const cyclic: any = { values: [1, 2] } + cyclic.self = cyclic + await expect(invoke(server, 'enforceRpcRequestBudgets', 'getSettings', [cyclic])).resolves.toBeUndefined() + + const deeplyNested: Record = {} + let cursor = deeplyNested + for (let index = 0; index < 65; index += 1) { + const child: Record = {} + cursor.child = child + cursor = child + } + await expect(invoke(server, 'enforceRpcRequestBudgets', 'getSettings', [deeplyNested])).rejects.toThrow( + 'nesting exceeds 64 levels' + ) + }) + + test('bounds nested request arrays and serialized RPC responses', async () => { + const server = makeServer( + { getSettings: jest.fn(() => ({ value: 'x'.repeat(1_000) })) }, + { maxRpcArrayItems: 2, maxRpcResponseBytes: 128 } + ) + await expect(invoke(server, 'enforceRpcRequestBudgets', 'getSettings', [[1, 2, 3]])).rejects.toThrow( + 'must not exceed 2 items' + ) + + const captured = makeResponse() + await invoke( + server, + 'handleRpcRequest', + makeRequest({ jsonrpc: '2.0', method: 'getSettings', params: [], id: 11 }), + captured.response + ) + expect(captured.statusCode).toBe(413) + expect(captured.body).toMatchObject({ + error: { code: -32005 } + }) + }) + test('rejects an RPC request without a valid authenticated identity', async () => { const server = makeServer() const body = { diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.test.ts index 1e169aee3..3d59d3882 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.test.ts @@ -5,13 +5,18 @@ import { concurrencyLimit, configureHttpServer, corsPolicy, + initialDoubleSlashCompatibility, + profileValue, readAllowedOrigins, readBodyLimitBytes, readCorsOriginSetting, + readResourceLimit, + readResourceProfile, + responseSizeLimit, securityHeaders } from './edgePolicy' -async function listen (app: express.Express): Promise<{ +async function listen(app: express.Express): Promise<{ server: Server origin: string }> { @@ -25,7 +30,7 @@ async function listen (app: express.Express): Promise<{ return { server, origin: `http://127.0.0.1:${address.port}` } } -async function close (server: Server): Promise { +async function close(server: Server): Promise { await new Promise((resolve, reject) => { server.close(error => { if (error != null) reject(error) @@ -47,10 +52,12 @@ describe('shared service edge policy', () => { delete process.env.CORS_MODE delete process.env.CORS_ALLOWED_ORIGINS const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -78,8 +85,9 @@ describe('shared service edge policy', () => { }) expect(preflight.status).toBe(204) expect(preflight.headers.get('access-control-allow-origin')).toBe('*') - expect(preflight.headers.get('access-control-allow-headers')) - .toContain('X-BSV-Action-Batch-Encoding') + expect(preflight.headers.get('access-control-allow-headers')).toContain( + 'X-BSV-Action-Batch-Encoding' + ) } finally { await close(server) } @@ -88,10 +96,12 @@ describe('shared service edge policy', () => { it('allows only explicitly configured browser origins', async () => { process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -124,10 +134,12 @@ describe('shared service edge policy', () => { delete process.env.TEST_CORS_ALLOWED_ORIGINS delete process.env.CORS_ALLOWED_ORIGINS const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -146,10 +158,12 @@ describe('shared service edge policy', () => { it('answers allowed preflight without wildcard policy', async () => { process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET', 'POST'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET', 'POST'] + }) + ) const { server, origin } = await listen(app) try { @@ -170,16 +184,20 @@ describe('shared service edge policy', () => { it('rejects wildcard and malformed origin configuration', () => { process.env.TEST_CORS_ALLOWED_ORIGINS = '*' - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })).toThrow(/wildcard/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ).toThrow(/wildcard/) process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example/path' - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })).toThrow(/without paths/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ).toThrow(/without paths/) }) it('validates the complete CORS mode configuration matrix', () => { @@ -202,14 +220,8 @@ describe('shared service edge policy', () => { process.env.TEST_CORS_MODE = 'allowlist' process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example, https://wallet.example, https://wui.example' - expect(readAllowedOrigins('TEST')).toEqual([ - 'https://wallet.example', - 'https://wui.example' - ]) - expect(readCorsOriginSetting('TEST')).toEqual([ - 'https://wallet.example', - 'https://wui.example' - ]) + expect(readAllowedOrigins('TEST')).toEqual(['https://wallet.example', 'https://wui.example']) + expect(readCorsOriginSetting('TEST')).toEqual(['https://wallet.example', 'https://wui.example']) delete process.env.TEST_CORS_MODE delete process.env.TEST_CORS_ALLOWED_ORIGINS @@ -217,21 +229,27 @@ describe('shared service edge policy', () => { }) it('validates explicit origin and credential options', () => { - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowedOrigins: ['null'] - })).toThrow(/opaque/) - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowedOrigins: ['not an origin'] - })).toThrow(/invalid origin/) - expect(() => corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'], - allowCredentials: true - })).toThrow(/cookie credentials/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowedOrigins: ['null'] + }) + ).toThrow(/opaque/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowedOrigins: ['not an origin'] + }) + ).toThrow(/invalid origin/) + expect(() => + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'], + allowCredentials: true + }) + ).toThrow(/cookie credentials/) const disabled = corsPolicy({ environmentPrefix: 'TEST', @@ -277,11 +295,7 @@ describe('shared service edge policy', () => { sendStatus: jest.fn() } const next = jest.fn() - middleware( - { get: () => 'https://wallet.example', method: 'GET' } as any, - response as any, - next - ) + middleware({ get: () => 'https://wallet.example', method: 'GET' } as any, response as any, next) expect(headers.get('Vary')).toBe('Accept-Encoding, Origin') expect(headers.get('Access-Control-Allow-Origin')).toBe('https://wallet.example') expect(headers.get('Access-Control-Allow-Credentials')).toBe('true') @@ -296,10 +310,12 @@ describe('shared service edge policy', () => { process.env.TEST_CORS_MODE = 'allowlist' process.env.TEST_CORS_ALLOWED_ORIGINS = 'https://wallet.example' const app = express() - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -342,15 +358,17 @@ describe('shared service edge policy', () => { process.env.TEST_STRICT_TRANSPORT_SECURITY = 'false' const app = express() app.enable('trust proxy') - app.use(securityHeaders({ - environmentPrefix: 'TEST', - contentSecurityPolicy: "default-src 'none'", - crossOriginResourcePolicy: 'same-origin', - crossOriginOpenerPolicy: 'same-origin', - frameOptions: 'DENY', - permissionsPolicy: 'camera=()', - strictTransportSecurity: true - })) + app.use( + securityHeaders({ + environmentPrefix: 'TEST', + contentSecurityPolicy: "default-src 'none'", + crossOriginResourcePolicy: 'same-origin', + crossOriginOpenerPolicy: 'same-origin', + frameOptions: 'DENY', + permissionsPolicy: 'camera=()', + strictTransportSecurity: true + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -375,10 +393,12 @@ describe('shared service edge policy', () => { delete process.env.TEST_CORS_ALLOWED_ORIGINS const app = express() app.use(securityHeaders({ environmentPrefix: 'TEST' })) - app.use(corsPolicy({ - environmentPrefix: 'TEST', - methods: ['GET'] - })) + app.use( + corsPolicy({ + environmentPrefix: 'TEST', + methods: ['GET'] + }) + ) app.get('/', (_req, res) => res.json({ ok: true })) const { server, origin } = await listen(app) @@ -389,8 +409,9 @@ describe('shared service edge policy', () => { expect(response.status).toBe(200) expect(response.headers.get('access-control-allow-origin')).toBe('*') expect(response.headers.get('access-control-allow-credentials')).toBeNull() - expect(response.headers.get('content-security-policy')) - .toBe("default-src 'self'; connect-src https:") + expect(response.headers.get('content-security-policy')).toBe( + "default-src 'self'; connect-src https:" + ) } finally { await close(server) } @@ -519,7 +540,9 @@ describe('shared service edge policy', () => { app.use(concurrencyLimit('TEST', 10)) let releaseFirst: (() => void) | undefined app.get('/', async (_req, res) => { - await new Promise(resolve => { releaseFirst = resolve }) + await new Promise(resolve => { + releaseFirst = resolve + }) res.json({ ok: true }) }) const { server, origin } = await listen(app) @@ -528,7 +551,8 @@ describe('shared service edge policy', () => { headersTimeoutMs: 10_000, keepAliveTimeoutMs: 5_000, socketTimeoutMs: 30_000, - maxRequestsPerSocket: 100 + maxRequestsPerSocket: 100, + maxConnections: 50 }) try { @@ -545,8 +569,156 @@ describe('shared service edge policy', () => { expect(server.headersTimeout).toBe(10_000) expect(server.keepAliveTimeout).toBe(5_000) expect(server.maxRequestsPerSocket).toBe(100) + expect(server.maxConnections).toBe(50) } finally { await close(server) } }) + + it('honors explicit unlimited body, response, concurrency, and connection limits', () => { + process.env.TEST_MAX_BODY_BYTES = '-1' + expect(readBodyLimitBytes('TEST', 256)).toBe(Number.MAX_SAFE_INTEGER) + + process.env.TEST_MAX_RESPONSE_BYTES = 'unlimited' + const responseNext = jest.fn() + responseSizeLimit('TEST', 256)({} as any, {} as any, responseNext) + expect(responseNext).toHaveBeenCalledTimes(1) + + process.env.TEST_MAX_CONCURRENT_REQUESTS = '-1' + const concurrencyNext = jest.fn() + concurrencyLimit('TEST', 8)({} as any, {} as any, concurrencyNext) + expect(concurrencyNext).toHaveBeenCalledTimes(1) + + process.env.TEST_MAX_CONNECTIONS = '-1' + const server = { + setTimeout: jest.fn() + } as unknown as Server + configureHttpServer(server, 'TEST', { + requestTimeoutMs: 30_000, + headersTimeoutMs: 10_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 30_000, + maxRequestsPerSocket: 100, + maxConnections: 50 + }) + expect(server.maxConnections).toBe(Number.MAX_SAFE_INTEGER) + }) + + it('rejects invalid positive HTTP server settings', () => { + process.env.TEST_REQUEST_TIMEOUT_MS = '0' + const server = { + setTimeout: jest.fn() + } as unknown as Server + + expect(() => + configureHttpServer(server, 'TEST', { + requestTimeoutMs: 30_000, + headersTimeoutMs: 10_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 30_000, + maxRequestsPerSocket: 100, + maxConnections: 50 + }) + ).toThrow(/positive integer/) + }) + + it('selects tested resource profiles and explicit operator limits', () => { + expect(readResourceProfile('TEST')).toBe('standard') + process.env.TEST_RESOURCE_PROFILE = 'high-throughput' + expect(readResourceProfile('TEST')).toBe('high-throughput') + expect(profileValue('small', { small: 1, standard: 2, highThroughput: 3 })).toBe(1) + expect(profileValue('standard', { small: 1, standard: 2, highThroughput: 3 })).toBe(2) + expect(profileValue('high-throughput', { small: 1, standard: 2, highThroughput: 3 })).toBe(3) + + process.env.TEST_MAX_ITEMS = '1000' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(1_000) + process.env.TEST_MAX_ITEMS = '-1' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(-1) + process.env.TEST_MAX_ITEMS = 'unlimited' + expect(readResourceLimit('TEST', 'MAX_ITEMS', 100)).toBe(-1) + process.env.TEST_MAX_ITEMS = '0' + expect(() => readResourceLimit('TEST', 'MAX_ITEMS', 100)).toThrow(/positive integer/) + + process.env.TEST_RESOURCE_PROFILE = 'oversized' + expect(() => readResourceProfile('TEST')).toThrow(/small, standard, or high-throughput/) + }) + + it('tolerates only repeated initial slashes for compatibility', () => { + const next = jest.fn() + const request = { url: '///auth/start?mode=test' } + initialDoubleSlashCompatibility(request as any, {} as any, next) + expect(request.url).toBe('/auth/start?mode=test') + expect(next).toHaveBeenCalledTimes(1) + + const interior = { url: '/auth//start' } + initialDoubleSlashCompatibility(interior as any, {} as any, jest.fn()) + expect(interior.url).toBe('/auth//start') + }) + + it('rejects materialized responses above the configured byte budget', async () => { + process.env.TEST_MAX_RESPONSE_BYTES = '128' + const app = express() + app.use(responseSizeLimit('TEST', 1024)) + app.get('/small', (_req, res) => res.json({ ok: true })) + app.get('/large', (_req, res) => res.json({ value: 'x'.repeat(512) })) + const { server, origin } = await listen(app) + + try { + const small = await fetch(`${origin}/small`) + expect(small.status).toBe(200) + await expect(small.json()).resolves.toEqual({ ok: true }) + + const large = await fetch(`${origin}/large`) + expect(large.status).toBe(413) + await expect(large.json()).resolves.toMatchObject({ code: 'ERR_RESPONSE_TOO_LARGE' }) + } finally { + await close(server) + } + }) + + it.each([ + ['send', '12345', undefined], + ['send', Buffer.from('12345'), undefined], + ['send', new Uint8Array([1, 2, 3, 4, 5]), undefined], + ['send', { value: '12345' }, undefined], + ['end', '12345', 'utf8'], + ['end', Buffer.from('12345'), undefined], + ['end', new Uint8Array([1, 2, 3, 4, 5]), undefined], + ['end', { value: '12345' }, undefined] + ])('bounds every materialized %s response shape', (method, value, encoding) => { + process.env.TEST_MAX_RESPONSE_BYTES = '4' + let response: any + const originalEnd = jest.fn(() => response) + const originalSend = jest.fn((chunk: unknown) => { + response.end(chunk) + return response + }) + const originalJson = jest.fn((body: unknown) => { + response.send(JSON.stringify(body)) + return response + }) + response = { + status: jest.fn(() => response), + json: originalJson, + send: originalSend, + end: originalEnd + } + const next = jest.fn() + responseSizeLimit('TEST', 256)({} as any, response, next) + + if (method === 'send') response.send(value) + else response.end(value, encoding) + + expect(response.status).toHaveBeenCalledWith(413) + expect(originalJson).toHaveBeenCalledWith( + expect.objectContaining({ code: 'ERR_RESPONSE_TOO_LARGE' }) + ) + expect(originalSend).toHaveBeenCalled() + expect(originalEnd).toHaveBeenCalled() + + response.json({ ignored: true }) + response.send('ignored') + response.end('ignored') + expect(response.status).toHaveBeenCalledTimes(1) + }) }) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts index 702aa22d9..71bdc02c2 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts @@ -1,11 +1,6 @@ // Synchronized by scripts/sync-service-edge-policy.mjs. Edit // infra/wab/src/security/edgePolicy.ts, then run the sync command. -import type { - NextFunction, - Request, - RequestHandler, - Response -} from 'express' +import type { NextFunction, Request, RequestHandler, Response } from 'express' import type { Server } from 'node:http' const MAX_BODY_BYTES = 512 * 1024 * 1024 @@ -85,13 +80,19 @@ export interface HttpServerPolicyDefaults { keepAliveTimeoutMs: number socketTimeoutMs: number maxRequestsPerSocket: number + /** Open TCP/WebSocket connections retained by one process. Default: 1,000. */ + maxConnections?: number } -function readPositiveInteger ( - name: string, - fallback: number, - maximum: number -): number { +export type ResourceProfileName = 'small' | 'standard' | 'high-throughput' + +export interface ResourceProfileValues { + small: number + standard: number + highThroughput: number +} + +function readPositiveInteger(name: string, fallback: number, maximum: number): number { const value = process.env[name] if (value == null || value.trim() === '') return fallback if (!/^[1-9]\d*$/.test(value)) { @@ -104,17 +105,68 @@ function readPositiveInteger ( return parsed } -function readCsv (name: string, fallback: string[] = []): string[] { +/** + * Reads an operator resource limit. `-1` and `unlimited` are explicit opt-outs; + * omitting the setting always retains the service's tested safe default. + */ +export function readResourceLimit( + environmentPrefix: string, + suffix: string, + fallback: number, + maximum: number = Number.MAX_SAFE_INTEGER +): number { + const name = `${environmentPrefix}_${suffix}` const value = process.env[name] if (value == null || value.trim() === '') return fallback - const values = value.split(',').map(item => item.trim()).filter(Boolean) + const normalized = value.trim().toLowerCase() + if (normalized === '-1' || normalized === 'unlimited') return -1 + if (!/^[1-9]\d*$/.test(normalized)) { + throw new Error(`${name} must be -1, unlimited, or a positive integer`) + } + const parsed = Number(normalized) + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + throw new Error(`${name} must not exceed ${maximum}`) + } + return parsed +} + +export function readResourceProfile( + environmentPrefix: string, + fallback: ResourceProfileName = 'standard' +): ResourceProfileName { + const prefixed = process.env[`${environmentPrefix}_RESOURCE_PROFILE`] + const value = + (prefixed == null || prefixed.trim() === '' ? process.env.RESOURCE_PROFILE : prefixed) + ?.trim() + .toLowerCase() ?? fallback + if (!['small', 'standard', 'high-throughput'].includes(value)) { + throw new Error( + `${environmentPrefix}_RESOURCE_PROFILE must be small, standard, or high-throughput` + ) + } + return value as ResourceProfileName +} + +export function profileValue(profile: ResourceProfileName, values: ResourceProfileValues): number { + if (profile === 'small') return values.small + if (profile === 'high-throughput') return values.highThroughput + return values.standard +} + +function readCsv(name: string, fallback: string[] = []): string[] { + const value = process.env[name] + if (value == null || value.trim() === '') return fallback + const values = value + .split(',') + .map(item => item.trim()) + .filter(Boolean) if (values.includes('*')) { throw new Error(`${name} must contain explicit values; wildcard "*" is not allowed`) } return [...new Set(values)] } -function normalizeOrigin (origin: string, name: string): string { +function normalizeOrigin(origin: string, name: string): string { if (origin === 'null') throw new Error(`${name} must not contain the opaque "null" origin`) let parsed: URL try { @@ -130,26 +182,26 @@ function normalizeOrigin (origin: string, name: string): string { parsed.search !== '' || parsed.hash !== '' ) { - throw new Error(`${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments`) + throw new Error( + `${name} must contain HTTP(S) origins without paths, credentials, queries, or fragments` + ) } return parsed.origin } -export function readAllowedOrigins (environmentPrefix: string): string[] { +export function readAllowedOrigins(environmentPrefix: string): string[] { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` const prefixedValue = process.env[originVariable] - const sourceVariable = prefixedValue != null && prefixedValue.trim() !== '' - ? originVariable - : 'CORS_ALLOWED_ORIGINS' - return readCsv(sourceVariable) - .map(origin => normalizeOrigin(origin, sourceVariable)) + const sourceVariable = + prefixedValue != null && prefixedValue.trim() !== '' ? originVariable : 'CORS_ALLOWED_ORIGINS' + return readCsv(sourceVariable).map(origin => normalizeOrigin(origin, sourceVariable)) } -function resolveCorsPolicy ( +function resolveCorsPolicy( environmentPrefix: string, configuredOrigins: string[] | undefined, defaultMode: CorsMode -): { mode: CorsMode, origins: string[] } { +): { mode: CorsMode; origins: string[] } { const originVariable = `${environmentPrefix}_CORS_ALLOWED_ORIGINS` if (configuredOrigins !== undefined) { const origins = configuredOrigins.map(origin => normalizeOrigin(origin, originVariable)) @@ -162,10 +214,10 @@ function resolveCorsPolicy ( const origins = readAllowedOrigins(environmentPrefix) const prefixedMode = process.env[`${environmentPrefix}_CORS_MODE`]?.trim() const rawMode = ( - prefixedMode !== undefined && prefixedMode !== '' - ? prefixedMode - : process.env.CORS_MODE ?? '' - ).trim().toLowerCase() + prefixedMode !== undefined && prefixedMode !== '' ? prefixedMode : (process.env.CORS_MODE ?? '') + ) + .trim() + .toLowerCase() let mode: string = rawMode if (mode === '') { mode = origins.length > 0 ? 'allowlist' : defaultMode @@ -186,7 +238,7 @@ function resolveCorsPolicy ( * Socket.IO and similar transports can consume the same public/allowlist/ * disabled policy as the HTTP middleware. */ -export function readCorsOriginSetting ( +export function readCorsOriginSetting( environmentPrefix: string, defaultMode: CorsMode = 'public' ): '*' | string[] { @@ -196,7 +248,7 @@ export function readCorsOriginSetting ( return policy.origins } -function appendVary (res: Response, value: string): void { +function appendVary(res: Response, value: string): void { const current = res.getHeader('Vary') const values = new Set( (Array.isArray(current) ? current.join(',') : String(current ?? '')) @@ -214,7 +266,7 @@ function appendVary (res: Response, value: string): void { * opt into an exact allowlist or disable cross-origin browser calls with * _CORS_MODE. */ -export function corsPolicy (options: CorsPolicyOptions): RequestHandler { +export function corsPolicy(options: CorsPolicyOptions): RequestHandler { const policy = resolveCorsPolicy( options.environmentPrefix, options.allowedOrigins, @@ -303,7 +355,7 @@ export function corsPolicy (options: CorsPolicyOptions): RequestHandler { } } -function readHeaderSetting ( +function readHeaderSetting( environmentPrefix: string | undefined, suffix: string ): string | undefined { @@ -315,7 +367,7 @@ function readHeaderSetting ( return value.trim() } -function readOptionalHeader ( +function readOptionalHeader( environmentPrefix: string | undefined, suffix: string, allowedValues?: string[] @@ -331,7 +383,7 @@ function readOptionalHeader ( return value } -function readOptionalBoolean ( +function readOptionalBoolean( environmentPrefix: string | undefined, suffix: string ): boolean | undefined { @@ -342,35 +394,29 @@ function readOptionalBoolean ( throw new Error(`${environmentPrefix ?? 'SERVICE'}_${suffix} must be true or false`) } -export function securityHeaders ( - options: SecurityHeadersOptions = {} -): RequestHandler { +export function securityHeaders(options: SecurityHeadersOptions = {}): RequestHandler { const contentSecurityPolicy = readOptionalHeader(options.environmentPrefix, 'CONTENT_SECURITY_POLICY') ?? options.contentSecurityPolicy ?? "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" const crossOriginResourcePolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_RESOURCE_POLICY', - ['same-origin', 'same-site', 'cross-origin'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_RESOURCE_POLICY', [ + 'same-origin', + 'same-site', + 'cross-origin' + ]) ?? options.crossOriginResourcePolicy ?? 'cross-origin' const crossOriginOpenerPolicy = - readOptionalHeader( - options.environmentPrefix, - 'CROSS_ORIGIN_OPENER_POLICY', - ['same-origin', 'same-origin-allow-popups', 'unsafe-none'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'CROSS_ORIGIN_OPENER_POLICY', [ + 'same-origin', + 'same-origin-allow-popups', + 'unsafe-none' + ]) ?? options.crossOriginOpenerPolicy ?? 'same-origin' const frameOptions = - readOptionalHeader( - options.environmentPrefix, - 'FRAME_OPTIONS', - ['DENY', 'SAMEORIGIN'] - ) ?? + readOptionalHeader(options.environmentPrefix, 'FRAME_OPTIONS', ['DENY', 'SAMEORIGIN']) ?? options.frameOptions ?? 'DENY' const permissionsPolicy = @@ -401,29 +447,132 @@ export function securityHeaders ( if (contentSecurityPolicy !== false) { res.setHeader('Content-Security-Policy', contentSecurityPolicy) } - if ( - strictTransportSecurity && - req.secure - ) { + if (strictTransportSecurity && req.secure) { res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') } next() } } -export function readBodyLimitBytes ( +export function readBodyLimitBytes( environmentPrefix: string, fallback: number, maximum: number = MAX_BODY_BYTES ): number { - return readPositiveInteger(`${environmentPrefix}_MAX_BODY_BYTES`, fallback, maximum) + const limit = readResourceLimit(environmentPrefix, 'MAX_BODY_BYTES', fallback, maximum) + // raw-body/body-parser interpret negative numbers as a zero-ish ceiling. + // A deliberately unlimited operator setting therefore maps to the largest + // exactly representable byte count accepted by those APIs. + return limit === -1 ? Number.MAX_SAFE_INTEGER : limit +} + +/** + * Preserve compatibility with clients that accidentally emit two or more + * initial slashes. Only the initial slash run is normalized; the remainder of + * the path and query string is unchanged. + */ +export function initialDoubleSlashCompatibility( + req: Request, + _res: Response, + next: NextFunction +): void { + if (req.url.startsWith('//')) req.url = req.url.replace(/^\/{2,}/, '/') + next() +} + +function responseChunkByteLength(chunk: unknown, encoding: unknown): number { + if (typeof chunk === 'string') { + const bufferEncoding = typeof encoding === 'string' ? (encoding as BufferEncoding) : 'utf8' + return Buffer.byteLength(chunk, bufferEncoding) + } + if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) return chunk.byteLength + return Buffer.byteLength(JSON.stringify(chunk) ?? '', 'utf8') +} + +/** + * Bounds materialized JSON/text/binary Express responses before downstream + * authentication middleware signs or serializes them again. Streaming + * endpoints must enforce their own byte budget while producing chunks. + */ +export function responseSizeLimit( + environmentPrefix: string, + fallback: number, + maximum: number = MAX_BODY_BYTES +): RequestHandler { + const limit = readResourceLimit(environmentPrefix, 'MAX_RESPONSE_BYTES', fallback, maximum) + if (limit === -1) return (_req, _res, next) => next() + + return (_req: Request, res: Response, next: NextFunction): void => { + const originalStatus = res.status.bind(res) + const originalJson = res.json.bind(res) + const originalSend = res.send.bind(res) + const originalEnd = res.end.bind(res) + let rejected = false + let rejecting = false + + const tooLarge = (byteLength: number): boolean => byteLength > limit + const reject = (): Response => { + if (rejected) return res + rejected = true + rejecting = true + originalStatus(413) + const response = originalJson({ + status: 'error', + code: 'ERR_RESPONSE_TOO_LARGE', + description: 'The requested response exceeds the configured service limit.' + }) + rejecting = false + return response + } + + res.json = ((value: unknown): Response => { + if (rejecting) return originalJson(value) + if (rejected) return res + const serialized = JSON.stringify(value) ?? '' + if (tooLarge(Buffer.byteLength(serialized, 'utf8'))) return reject() + return originalJson(value) + }) as Response['json'] + + res.send = ((value: unknown): Response => { + if (rejecting) return originalSend(value as never) + if (rejected) return res + let byteLength: number + if (typeof value === 'string') byteLength = Buffer.byteLength(value, 'utf8') + else if (Buffer.isBuffer(value) || value instanceof Uint8Array) byteLength = value.byteLength + else byteLength = Buffer.byteLength(JSON.stringify(value) ?? '', 'utf8') + if (tooLarge(byteLength)) return reject() + return originalSend(value as never) + }) as Response['send'] + + res.end = ((chunk?: unknown, encoding?: unknown, callback?: unknown): Response => { + if (rejecting) { + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + } + if (rejected) return res + if (chunk != null) { + const byteLength = responseChunkByteLength(chunk, encoding) + if (tooLarge(byteLength)) return reject() + } + return originalEnd( + chunk as never, + encoding as never, + callback as never + ) as unknown as Response + }) as Response['end'] + + next() + } } /** * Convert body-parser/Express parser failures into stable, non-sensitive * protocol errors. Install immediately after all body parsers. */ -export function bodyParserErrorHandler ( +export function bodyParserErrorHandler( error: unknown, _req: Request, res: Response, @@ -466,15 +615,14 @@ export function bodyParserErrorHandler ( * unbounded in-flight application work. Distributed/global quotas remain the * responsibility of the deployment's shared rate-limit store or gateway. */ -export function concurrencyLimit ( - environmentPrefix: string, - fallback: number -): RequestHandler { - const maximum = readPositiveInteger( - `${environmentPrefix}_MAX_CONCURRENT_REQUESTS`, +export function concurrencyLimit(environmentPrefix: string, fallback: number): RequestHandler { + const maximum = readResourceLimit( + environmentPrefix, + 'MAX_CONCURRENT_REQUESTS', fallback, MAX_CONCURRENT_REQUESTS ) + if (maximum === -1) return (_req, _res, next) => next() let active = 0 return (_req: Request, res: Response, next: NextFunction): void => { @@ -501,7 +649,7 @@ export function concurrencyLimit ( } } -export function configureHttpServer ( +export function configureHttpServer( server: Server, environmentPrefix: string, defaults: HttpServerPolicyDefaults @@ -535,6 +683,13 @@ export function configureHttpServer ( defaults.maxRequestsPerSocket, 1_000_000 ) + const maxConnections = readResourceLimit( + environmentPrefix, + 'MAX_CONNECTIONS', + defaults.maxConnections ?? 1_000, + 1_000_000 + ) + server.maxConnections = maxConnections === -1 ? Number.MAX_SAFE_INTEGER : maxConnections if (typeof server.setTimeout === 'function') { server.setTimeout(socketTimeoutMs) } diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 6f0a36bc4..a795ff0cd 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -9,6 +9,7 @@ import { WERR_NOT_IMPLEMENTED } from '../../sdk/WERR_errors' export const AUTH_SESSION_MIGRATION = '2026-07-14-001 add shared auth sessions' export const MONITOR_CREATED_AT_INDEX_MIGRATION = '2026-07-14-002 add monitor created index' export const CREATE_ACTION_FUNDING_INDEX_MIGRATION = '2026-08-02-001 add createAction funding selection index' +export const PAYMENT_REPLAY_MIGRATION = '2026-08-04-001 add payment replay claims' interface Migration { up: (knex: Knex) => Promise @@ -30,7 +31,7 @@ export class KnexMigrations implements MigrationSource { * @param storageName human readable name for this storage instance * @param maxOutputScriptLength limit for scripts kept in outputs table, longer scripts will be pulled from rawTx */ - constructor ( + constructor( public chain: Chain, public storageName: string, public storageIdentityKey: string, @@ -39,29 +40,29 @@ export class KnexMigrations implements MigrationSource { this.migrations = this.setupMigrations(chain, storageName, storageIdentityKey, maxOutputScriptLength) } - async getMigrations (): Promise { + async getMigrations(): Promise { return Object.keys(this.migrations).sort((a, b) => a.localeCompare(b)) } - getMigrationName (migration: string) { + getMigrationName(migration: string) { return migration } - async getMigration (migration: string): Promise { + async getMigration(migration: string): Promise { return this.migrations[migration] } - async getLatestMigration (): Promise { + async getLatestMigration(): Promise { const ms = await this.getMigrations() return ms.at(-1)! } - static async latestMigration (): Promise { + static async latestMigration(): Promise { const km = new KnexMigrations('test', 'dummy', '1'.repeat(64), 100) return await km.getLatestMigration() } - setupMigrations ( + setupMigrations( chain: string, storageName: string, storageIdentityKey: string, @@ -80,7 +81,7 @@ export class KnexMigrations implements MigrationSource { } migrations[AUTH_SESSION_MIGRATION] = { - async up (knex) { + async up(knex) { await knex.schema.createTable('auth_sessions', table => { table.string('sessionNonce', 64).primary() table.string('peerNonce', 64).nullable() @@ -94,18 +95,32 @@ export class KnexMigrations implements MigrationSource { table.index('expiresAt', 'idx_auth_sessions_expires') }) }, - async down (knex) { + async down(knex) { await knex.schema.dropTable('auth_sessions') } } + migrations[PAYMENT_REPLAY_MIGRATION] = { + async up(knex) { + await knex.schema.createTable('payment_replays', table => { + table.string('transactionId', 64).primary() + table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()) + table.timestamp('expiresAt').nullable() + table.index('expiresAt', 'idx_payment_replays_expires') + }) + }, + async down(knex) { + await knex.schema.dropTable('payment_replays') + } + } + migrations[MONITOR_CREATED_AT_INDEX_MIGRATION] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('monitor_events', table => { table.index('created_at', 'idx_monitor_events_created_at') }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('monitor_events', table => { table.dropIndex('created_at', 'idx_monitor_events_created_at') }) @@ -113,7 +128,7 @@ export class KnexMigrations implements MigrationSource { } migrations[CREATE_ACTION_FUNDING_INDEX_MIGRATION] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('outputs', table => { table.index( ['userId', 'basketId', 'spendable', 'spentBy', 'satoshis', 'outputId'], @@ -121,7 +136,7 @@ export class KnexMigrations implements MigrationSource { ) }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('outputs', table => { table.dropIndex( ['userId', 'basketId', 'spendable', 'spentBy', 'satoshis', 'outputId'], @@ -132,7 +147,7 @@ export class KnexMigrations implements MigrationSource { } migrations['2026-07-15-001 add action batch reservations and blobs'] = { - async up (knex) { + async up(knex) { const dbtype = await determineDBType(knex) await knex.schema.createTable('action_batches', table => { addTimeStamps(knex, table, dbtype) @@ -169,7 +184,7 @@ export class KnexMigrations implements MigrationSource { await knex.raw('ALTER TABLE action_batch_blobs MODIFY COLUMN bytes LONGBLOB') } }, - async down (knex) { + async down(knex) { await knex.schema.dropTable('action_batch_blobs') await knex.schema.dropTable('action_batch_outputs') await knex.schema.dropTable('action_batches') @@ -177,12 +192,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2026-07-26-001 retain prepared action batch manifests'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('action_batches', table => { table.text('manifest', 'longtext').nullable() }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('action_batches', table => { table.dropColumn('manifest') }) @@ -190,7 +205,7 @@ export class KnexMigrations implements MigrationSource { } migrations['2026-04-30-001 add wasBroadcast and rebroadcastAttempts to proven_tx_reqs'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('proven_tx_reqs', table => { table.boolean('wasBroadcast').notNullable().defaultTo(false) table.integer('rebroadcastAttempts').unsigned().notNullable().defaultTo(0) @@ -199,7 +214,7 @@ export class KnexMigrations implements MigrationSource { .whereIn('status', ['unmined', 'callback', 'unconfirmed', 'completed']) .update({ wasBroadcast: true }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('proven_tx_reqs', table => { table.dropColumn('rebroadcastAttempts') table.dropColumn('wasBroadcast') @@ -208,12 +223,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-10-13-001 add outputs spendable index'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('outputs', table => { table.index('spendable') }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('outputs', table => { table.dropIndex('spendable') }) @@ -221,7 +236,7 @@ export class KnexMigrations implements MigrationSource { } migrations['2026-02-27-001 add listOutputs path indexes'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('outputs', table => { table.index(['userId', 'spendable', 'outputId'], 'idx_outputs_user_spendable_outputid') table.index(['userId', 'basketId', 'spendable', 'outputId'], 'idx_outputs_user_basket_spendable_outputid') @@ -233,15 +248,12 @@ export class KnexMigrations implements MigrationSource { table.index(['transactionId', 'isDeleted'], 'idx_tx_labels_map_tx_deleted') }) }, - async down (knex) { + async down(knex) { // MySQL may discard the automatically-created userId index once one // of these wider indexes can support the foreign key. Recreate the // original index before removing both wider indexes. - if (await determineDBType(knex) === 'MySQL') { - const result = await knex.raw( - 'SHOW INDEX FROM ?? WHERE Key_name = ?', - ['outputs', 'outputs_userid_foreign'] - ) + if ((await determineDBType(knex)) === 'MySQL') { + const result = await knex.raw('SHOW INDEX FROM ?? WHERE Key_name = ?', ['outputs', 'outputs_userid_foreign']) const indexes = result[0] as unknown[] if (indexes.length === 0) { await knex.schema.alterTable('outputs', table => { @@ -263,22 +275,19 @@ export class KnexMigrations implements MigrationSource { } migrations['2026-02-27-002 add createAction path indexes'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('outputs', table => { table.index(['userId', 'basketId', 'spendable', 'satoshis'], 'idx_outputs_user_basket_spendable_satoshis') table.index(['spentBy'], 'idx_outputs_spentby') }) }, - async down (knex) { + async down(knex) { // MySQL may discard the automatically-created index that supports the // spentBy foreign key after this migration adds an equivalent named // index. Restore the original support index before removing ours so a // complete rollback remains possible. - if (await determineDBType(knex) === 'MySQL') { - const result = await knex.raw( - 'SHOW INDEX FROM ?? WHERE Key_name = ?', - ['outputs', 'outputs_spentby_foreign'] - ) + if ((await determineDBType(knex)) === 'MySQL') { + const result = await knex.raw('SHOW INDEX FROM ?? WHERE Key_name = ?', ['outputs', 'outputs_spentby_foreign']) const indexes = result[0] as unknown[] if (indexes.length === 0) { await knex.schema.alterTable('outputs', table => { @@ -294,12 +303,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-10-18-002 add proven_tx_reqs txid index'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('proven_tx_reqs', table => { table.index('txid') }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('proven_tx_reqs', table => { table.dropIndex('txid') }) @@ -307,12 +316,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-10-18-001 add transactions txid index'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('transactions', table => { table.index('txid') }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('transactions', table => { table.dropIndex('txid') }) @@ -320,12 +329,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-09-06-001 add proven txs blockHash index'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('proven_txs', table => { table.index('blockHash') }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('proven_txs', table => { table.dropIndex('blockHash') }) @@ -333,12 +342,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-05-13-001 add monitor events event index'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('monitor_events', table => { table.index('event') }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('monitor_events', table => { table.dropIndex('event') }) @@ -346,7 +355,7 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-03-03-001 descriptions to 2000'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('transactions', table => { table.string('description', 2048).alter() }) @@ -355,32 +364,32 @@ export class KnexMigrations implements MigrationSource { table.string('spendingDescription', 2048).alter() }) }, - async down (knex) {} + async down(knex) {} } migrations['2025-03-01-001 reset req history'] = { - async up (knex) { + async up(knex) { const storage = new StorageKnex({ ...StorageKnex.defaultOptions(), chain: chain as Chain, knex }) await storage.makeAvailable() - await knex.raw('update proven_tx_reqs set history = \'{}\'') + await knex.raw("update proven_tx_reqs set history = '{}'") }, - async down (knex) { + async down(knex) { // No way back... } } migrations['2025-02-28-001 derivations to 200'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('outputs', table => { table.string('derivationPrefix', 200).alter() table.string('derivationSuffix', 200).alter() }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('outputs', table => { table.string('derivationPrefix', 32).alter() table.string('derivationSuffix', 32).alter() @@ -389,7 +398,7 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-02-22-001 nonNULL activeStorage'] = { - async up (knex) { + async up(knex) { const storage = new StorageKnex({ ...StorageKnex.defaultOptions(), chain: chain as Chain, @@ -401,7 +410,7 @@ export class KnexMigrations implements MigrationSource { table.string('activeStorage').notNullable().alter() }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('users', table => { table.string('activeStorage').nullable().alter() }) @@ -409,12 +418,12 @@ export class KnexMigrations implements MigrationSource { } migrations['2025-01-21-001 add activeStorage to users'] = { - async up (knex) { + async up(knex) { await knex.schema.alterTable('users', table => { table.string('activeStorage', 130).nullable().defaultTo(null) }) }, - async down (knex) { + async down(knex) { await knex.schema.alterTable('users', table => { table.dropColumn('activeStorage') }) @@ -422,7 +431,7 @@ export class KnexMigrations implements MigrationSource { } migrations['2024-12-26-001 initial migration'] = { - async up (knex) { + async up(knex) { const dbtype = await determineDBType(knex) await knex.schema.createTable('proven_txs', table => { @@ -647,7 +656,7 @@ export class KnexMigrations implements MigrationSource { maxOutputScript: maxOutputScriptLength }) }, - async down (knex) { + async down(knex) { await knex.schema.dropTable('sync_states') await knex.schema.dropTable('settings') await knex.schema.dropTable('monitor_events') @@ -674,7 +683,7 @@ export class KnexMigrations implements MigrationSource { * @param knex * @returns {DBType} connected database engine variant */ -export async function determineDBType (knex: Knex): Promise { +export async function determineDBType(knex: Knex): Promise { try { const q = `SELECT CASE diff --git a/packages/wallet/wallet-toolbox/src/utility/utilityHelpers.ts b/packages/wallet/wallet-toolbox/src/utility/utilityHelpers.ts index a6f67092a..2ea8eddaa 100644 --- a/packages/wallet/wallet-toolbox/src/utility/utilityHelpers.ts +++ b/packages/wallet/wallet-toolbox/src/utility/utilityHelpers.ts @@ -25,6 +25,7 @@ export function toWalletNetwork(chain: Chain): WalletNetwork { case 'main': return 'mainnet' case 'test': + case 'stn': case 'ttn': case 'tstn': case 'mock': @@ -42,6 +43,7 @@ export function toLookupNetworkPreset(chain: Chain): 'mainnet' | 'testnet' | 'lo return 'mainnet' case 'test': return 'testnet' + case 'stn': case 'ttn': case 'tstn': case 'mock': diff --git a/scripts/benchmark-service-resource-profiles.mjs b/scripts/benchmark-service-resource-profiles.mjs new file mode 100644 index 000000000..61f4e9a7a --- /dev/null +++ b/scripts/benchmark-service-resource-profiles.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node + +import { readFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const root = fileURLToPath(new URL('..', import.meta.url)) +const manifestPath = `${root}/governance/service-resource-profiles.json` +const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) + +function sampleItem(targetBytes, index) { + const fixed = JSON.stringify({ + id: index, + created_at: '2026-08-04T00:00:00.000Z', + body: '' + }).length + return { + id: index, + created_at: '2026-08-04T00:00:00.000Z', + body: 'x'.repeat(Math.max(0, targetBytes - fixed)) + } +} + +function runChild(serviceName, profileName) { + const service = manifest.services[serviceName] + const values = service.values[profileName] + global.gc?.() + const before = process.memoryUsage() + const items = Array.from({ length: values.maxItems }, (_, index) => + sampleItem(service.representativeItemBytes, index) + ) + const json = JSON.stringify({ status: 'success', items }) + const authenticatedBytes = Buffer.from(json, 'utf8') + const reparsed = JSON.parse(json) + const after = process.memoryUsage() + if ( + reparsed.items.length !== values.maxItems || + authenticatedBytes.length !== Buffer.byteLength(json) + ) { + throw new Error('benchmark integrity check failed') + } + const representativeResponseBytes = authenticatedBytes.length + const modeledParallelBytes = + representativeResponseBytes * values.concurrency * manifest.measurement.duplicationFactor + process.stdout.write( + JSON.stringify({ + service: serviceName, + profile: profileName, + items: values.maxItems, + representativeResponseBytes, + responseCapBytes: values.maxResponseBytes, + measuredHeapDeltaBytes: Math.max(0, after.heapUsed - before.heapUsed), + measuredRssDeltaBytes: Math.max(0, after.rss - before.rss), + modeledParallelBytes, + minimumMemoryMiB: manifest.profiles[profileName].minimumMemoryMiB, + withinResponseCap: representativeResponseBytes <= values.maxResponseBytes, + withinModeledMemory: + modeledParallelBytes <= manifest.profiles[profileName].minimumMemoryMiB * 1024 * 1024 * 0.8 + }) + ) +} + +function runParent() { + const results = [] + for (const [serviceName, service] of Object.entries(manifest.services)) { + for (const profileName of Object.keys(service.values)) { + const heapMiB = Math.max( + 256, + Math.floor(manifest.profiles[profileName].minimumMemoryMiB * 0.75) + ) + const child = spawnSync( + process.execPath, + [ + '--expose-gc', + `--max-old-space-size=${heapMiB}`, + fileURLToPath(import.meta.url), + '--child', + serviceName, + profileName + ], + { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 } + ) + if (child.status !== 0) { + process.stderr.write(child.stderr) + throw new Error(`profile benchmark failed for ${serviceName}/${profileName}`) + } + results.push(JSON.parse(child.stdout)) + } + } + const failedCaps = results.filter(result => !result.withinResponseCap) + if (failedCaps.length > 0) { + const failedProfiles = failedCaps.map(r => `${r.service}/${r.profile}`).join(', ') + throw new Error(`representative pages exceed response caps: ${failedProfiles}`) + } + const report = { + generatedAt: new Date().toISOString(), + node: process.version, + platform: `${process.platform}/${process.arch}`, + duplicationFactor: manifest.measurement.duplicationFactor, + results + } + if (process.argv.includes('--check')) { + process.stdout.write(`Validated ${results.length} service resource-profile scenarios.\n`) + } else { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`) + } +} + +if (process.argv[2] === '--child') runChild(process.argv[3], process.argv[4]) +else runParent() diff --git a/scripts/ci-orchestration.test.mjs b/scripts/ci-orchestration.test.mjs index a60df4eb8..a3cb95a0e 100644 --- a/scripts/ci-orchestration.test.mjs +++ b/scripts/ci-orchestration.test.mjs @@ -8,6 +8,10 @@ import { REPOSITORY_ROOT } from './repository-health.mjs' const CI_PATH = join(REPOSITORY_ROOT, '.github/workflows/ci.yml') const CONFORMANCE_PATH = join(REPOSITORY_ROOT, '.github/workflows/conformance.yml') const RUNTIME_PATH = join(REPOSITORY_ROOT, '.github/workflows/container-runtime-contract.yml') +const WALLET_MOBILE_COVERAGE_PATH = join( + REPOSITORY_ROOT, + 'packages/wallet/wallet-toolbox/mobile/vitest.config.ts' +) function workflowJobBlocks(workflow) { const jobsMarker = '\njobs:\n' @@ -74,6 +78,18 @@ test('CI skips empty duplicate lanes without weakening the aggregate gate', () = assert.equal(workflow.match(/mongodb-memory-server binary cache warmed/g)?.length, 2) }) +test('CI contributes mobile and type-only wallet surfaces to aggregate patch coverage', () => { + const workflow = readFileSync(CI_PATH, 'utf8') + const mobileCoverage = readFileSync(WALLET_MOBILE_COVERAGE_PATH, 'utf8') + + assert.match(workflow, /pnpm --filter @bsv\/wallet-toolbox-mobile run test:coverage/) + assert.match(workflow, /name: coverage-wallet-mobile/) + assert.match(workflow, /^ - wallet-mobile-platform$/m) + for (const source of ['index.mobile.ts', 'BulkIngestorApi.ts', 'ChaintracksClientApi.ts']) { + assert.ok(mobileCoverage.includes(source), `${source} must be present in mobile LCOV`) + } +}) + test('CI push jobs survive intentionally skipped pull-request-only gates', () => { const workflow = readFileSync(CI_PATH, 'utf8') const jobs = Object.fromEntries(workflowJobBlocks(workflow).map(job => [job.name, job.source])) diff --git a/scripts/message-box-economics.mjs b/scripts/message-box-economics.mjs new file mode 100644 index 000000000..967621509 --- /dev/null +++ b/scripts/message-box-economics.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node + +function readNumber(name, fallback, minimum = 0) { + const raw = process.env[name] + if (raw == null || raw.trim() === '') return fallback + const value = Number(raw) + if (!Number.isFinite(value) || value < minimum) { + throw new Error(`${name} must be a finite number no less than ${minimum}`) + } + return value +} + +const model = { + monthlyFixedUsd: readNumber('MB_ECON_MONTHLY_FIXED_USD', 110), + bsvUsd: readNumber('MB_ECON_BSV_USD', 25, Number.EPSILON), + monthlyRequests: readNumber('MB_ECON_MONTHLY_REQUESTS', 10_000_000, 1), + operatingMargin: readNumber('MB_ECON_OPERATING_MARGIN', 0.25), + sendFraction: readNumber('MB_ECON_SEND_FRACTION', 0.5), + averageRecipients: readNumber('MB_ECON_AVERAGE_RECIPIENTS', 1, 1), + averageKiB: readNumber('MB_ECON_AVERAGE_KIB', 1), + averageRetentionMonths: readNumber('MB_ECON_AVERAGE_RETENTION_MONTHS', 1), + baseSatoshis: readNumber('MESSAGE_BOX_PRICE_BASE_SATOSHIS', 50), + perRecipientSatoshis: readNumber('MESSAGE_BOX_PRICE_PER_RECIPIENT_SATOSHIS', 5), + perKiBSatoshis: readNumber('MESSAGE_BOX_PRICE_PER_KIB_SATOSHIS', 5), + storageMiBMonthSatoshis: readNumber('MESSAGE_BOX_PRICE_STORAGE_MIB_MONTH_SATOSHIS', 1_000), + listPageSatoshis: readNumber('MESSAGE_BOX_PRICE_LIST_PAGE_SATOSHIS', 5) +} + +if (model.sendFraction > 1) throw new Error('MB_ECON_SEND_FRACTION must not exceed 1') + +const storagePerSend = + (model.averageKiB / 1024) * model.averageRetentionMonths * model.storageMiBMonthSatoshis +const sendVariable = + model.averageRecipients * model.perRecipientSatoshis + + model.averageKiB * model.perKiBSatoshis + + storagePerSend +const weightedVariable = + model.sendFraction * sendVariable + (1 - model.sendFraction) * model.listPageSatoshis +const requiredMonthlyUsd = model.monthlyFixedUsd * (1 + model.operatingMargin) +const requiredSatoshis = (requiredMonthlyUsd / model.bsvUsd) * 100_000_000 +const recommendedBaseSatoshis = Math.max( + 0, + Math.ceil(requiredSatoshis / model.monthlyRequests - weightedVariable) +) +const averageConfiguredPrice = model.baseSatoshis + weightedVariable +const projectedRevenueUsd = + (averageConfiguredPrice * model.monthlyRequests * model.bsvUsd) / 100_000_000 + +process.stdout.write( + `${JSON.stringify( + { + assumptions: model, + results: { + requiredMonthlyUsd, + weightedVariableSatoshis: weightedVariable, + recommendedBaseSatoshis, + configuredBaseSatoshis: model.baseSatoshis, + averageConfiguredPriceSatoshis: averageConfiguredPrice, + projectedRevenueUsd, + coversTarget: projectedRevenueUsd >= requiredMonthlyUsd + } + }, + null, + 2 + )}\n` +) diff --git a/scripts/repository-health.test.mjs b/scripts/repository-health.test.mjs index 78c627b06..219dcbd6a 100644 --- a/scripts/repository-health.test.mjs +++ b/scripts/repository-health.test.mjs @@ -47,7 +47,7 @@ test('workspace discovery exactly matches the 38-project registry', () => { [...projects.projects].map(project => project.path).sort() ) assert.deepEqual(validateProjectRegistry(projects, discovered), []) - assert.equal(projects.generatedArtifacts.length, 10) + assert.equal(projects.generatedArtifacts.length, 12) assert.ok(projects.generatedArtifacts.every(item => item.owner === 'ts-stack-maintainers')) assert.deepEqual(projects.dependencyAutomation.firstParty, { pattern: '@bsv/*', diff --git a/scripts/sonar-config.test.mjs b/scripts/sonar-config.test.mjs index 25d7aa5f6..3c001af82 100644 --- a/scripts/sonar-config.test.mjs +++ b/scripts/sonar-config.test.mjs @@ -38,7 +38,7 @@ test('Sonar Automatic Analysis excludes governed generated outputs but analyzes .split('\n') .filter(Boolean) const synchronizedCopies = registry.generatedArtifacts.filter(artifact => - artifact.generator.startsWith('scripts/sync-service-edge-policy.mjs') + artifact.generator.startsWith('scripts/sync-service-') ) const externallyGenerated = registry.generatedArtifacts.filter( artifact => !synchronizedCopies.includes(artifact) diff --git a/scripts/sync-service-runtime-copies.mjs b/scripts/sync-service-runtime-copies.mjs new file mode 100644 index 000000000..2c18b6c0c --- /dev/null +++ b/scripts/sync-service-runtime-copies.mjs @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' +import { readUtf8FileIfExists, writeUtf8FileAtomic } from './file-system.mjs' + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const policyPath = path.join(repositoryRoot, 'governance/service-runtime-copy-policy.json') +const policy = JSON.parse(fs.readFileSync(policyPath, 'utf8')) +const checkOnly = process.argv.includes('--check') + +if (policy.schemaVersion !== 1 || !Array.isArray(policy.copies) || policy.copies.length === 0) { + throw new Error('service runtime copy policy must declare at least one version 1 copy set') +} + +let drift = false +let synchronizedCount = 0 +for (const copySet of policy.copies) { + const canonicalPath = path.join(repositoryRoot, copySet.canonicalSource) + const canonical = fs.readFileSync(canonicalPath, 'utf8') + for (const relativePath of copySet.synchronizedSources) { + synchronizedCount += 1 + const absolutePath = path.join(repositoryRoot, relativePath) + if (readUtf8FileIfExists(absolutePath) === canonical) continue + drift = true + if (checkOnly) { + console.error(`${relativePath} differs from ${copySet.canonicalSource}`) + continue + } + writeUtf8FileAtomic(absolutePath, canonical) + console.log(`Synchronized ${relativePath}`) + } +} + +if (checkOnly && drift) { + console.error('Run `pnpm sync:service-runtime-copies` and commit the synchronized files.') + process.exitCode = 1 +} else if (checkOnly) { + console.log(`Service runtime sources are synchronized across ${synchronizedCount} copies.`) +} diff --git a/sonar-project.properties b/sonar-project.properties index 9898efebc..dc8d120ec 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -57,7 +57,9 @@ infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts,\ infra/message-box-server/src/security/edgePolicy.ts,\ infra/chaintracks-server/src/security/edgePolicy.ts,\ packages/overlays/overlay-express/src/security/edgePolicy.ts,\ -packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts +packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts,\ +infra/uhrp-server-cloud-bucket/src/resourceLimits.ts,\ +infra/wallet-infra/src/KnexPaymentReplayStore.ts # Keep CI and Automatic Analysis aligned on the same narrowly registered # compatibility exceptions. sonar.issue.ignore.multicriteria=werrProtocolNames,curveSingletonAlias,curveSingletonReturn,scriptOpcodeDispatch diff --git a/specs/messaging/authsocket-asyncapi.yaml b/specs/messaging/authsocket-asyncapi.yaml index 4cccf4bfd..774145ec8 100644 --- a/specs/messaging/authsocket-asyncapi.yaml +++ b/specs/messaging/authsocket-asyncapi.yaml @@ -198,9 +198,8 @@ components: type: string description: | Target room. Format: `-`. - The server extracts the `messageBoxType` by splitting on `-` and - taking the second part; it uses the authenticated sender key from - `authenticatedSockets`. + The server removes the exact recipient-key prefix and uses the + authenticated sender key from `authenticatedSockets`. message: type: object required: [messageId, recipient, body] @@ -217,16 +216,19 @@ components: WsSendMessageAckPayload: type: object description: | - Acknowledgement emitted by the server on `sendMessageAck-{roomId}` after - a successful `sendMessage`. Note: the event name is dynamic and includes - the room ID used in the originating request. - required: [status, messageId] + Acknowledgement emitted by the server on `sendMessageAck-{roomId}`. A + successful write includes `messageId`. An error includes `code`; paid + servers use `ERR_PAYMENT_REQUIRES_AUTHFETCH` so compatible clients retry + the send through the BRC-105 AuthFetch HTTP path. + required: [status] properties: status: type: string - enum: [success] + enum: [success, error] messageId: type: string + code: + type: string WsSendMessageBroadcastPayload: type: object @@ -385,7 +387,10 @@ channels: sendMessage: address: sendMessage description: | - Client sends a message to a recipient via WebSocket. The server: + Client sends a message to a recipient via WebSocket when operator + monetization is disabled. Paid servers return an error acknowledgement + that instructs current clients to use their AuthFetch HTTP fallback. The + unpriced WebSocket path: 1. Validates the sender is authenticated. 2. Validates `roomId` and `message`. 3. Creates the message box if it does not exist. @@ -401,9 +406,10 @@ channels: sendMessageAck: address: "sendMessageAck-{roomId}" description: | - Per-room acknowledgement emitted to the sender only after the message - is stored. The event name is `sendMessageAck-` where `roomId` - matches the value in the originating `sendMessage` payload. + Per-room acknowledgement emitted to the sender after the message is + stored or when the request must fall back to AuthFetch. The event name is + `sendMessageAck-` where `roomId` matches the value in the + originating `sendMessage` payload. parameters: roomId: description: The room ID from the originating sendMessage request. diff --git a/specs/messaging/message-box-http.yaml b/specs/messaging/message-box-http.yaml index 85cf882d1..c26055acf 100644 --- a/specs/messaging/message-box-http.yaml +++ b/specs/messaging/message-box-http.yaml @@ -500,6 +500,11 @@ paths: minimum: 0 maximum: 100000 default: 0 + skip: + type: integer + minimum: 0 + maximum: 100000 + description: Compatibility alias for offset; both must match if supplied together. responses: '200': description: Messages retrieved successfully (array may be empty). @@ -507,7 +512,7 @@ paths: application/json: schema: type: object - required: [status, messages, limit, offset, hasMore] + required: [status, messages, limit, offset, nextOffset, hasMore] properties: status: type: string @@ -521,6 +526,9 @@ paths: type: integer offset: type: integer + nextOffset: + type: integer + description: Offset for the next page; unchanged when the result is empty. hasMore: type: boolean '400':