From 31234e65ce760a0a05ddfa69d9b1992aa8898039 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Mon, 27 Jul 2026 22:29:41 -0700 Subject: [PATCH 1/4] feat: govern and harden service operations --- docs/reference/index.md | 1 + docs/reference/service-operations.md | 96 +++ docs/reference/stack-facts.md | 14 +- governance/service-operations.json | 113 +++ infra/chaintracks-server/Dockerfile | 3 + infra/chaintracks-server/package-lock.json | 4 +- infra/chaintracks-server/package.json | 2 +- infra/message-box-server/Dockerfile | 3 + infra/message-box-server/package-lock.json | 4 +- infra/message-box-server/package.json | 2 +- .../message-box-server/src/config/firebase.ts | 135 ++-- ...2025-01-31-001-notification-permissions.ts | 4 +- .../src/routes/permissions/setPermission.ts | 53 +- infra/overlay-server/Dockerfile | 3 + infra/overlay-server/deploy/README.md | 30 + .../overlay-server/deploy/app-configmap.yaml | 10 + .../overlay-server/deploy/app-deployment.yaml | 91 ++- infra/overlay-server/deploy/app-service.yaml | 6 +- .../deploy/janitor-cronjob.yaml | 31 +- .../deploy/mongodb-deployment.yaml | 30 +- .../deploy/mysql-deployment.yaml | 40 +- infra/overlay-server/package-lock.json | 4 +- infra/overlay-server/package.json | 2 +- infra/overlay-server/src/index.ts | 759 +++++++++--------- infra/uhrp-server-basic/Dockerfile | 3 + infra/uhrp-server-basic/package-lock.json | 4 +- infra/uhrp-server-basic/package.json | 2 +- infra/uhrp-server-basic/src/index.ts | 285 ++++--- infra/uhrp-server-basic/src/routes/list.ts | 25 +- infra/uhrp-server-basic/src/routes/renew.ts | 94 +-- infra/uhrp-server-basic/src/serviceHealth.ts | 24 + .../src/utils/getMetadata.ts | 25 +- .../src/utils/getPriceForFile.ts | 33 +- .../src/utils/getUploadURL.ts | 4 +- .../src/utils/mimeTypeMiddleware.ts | 91 ++- .../test/serviceHealth.test.js | 40 + infra/uhrp-server-cloud-bucket/Dockerfile | 3 + infra/uhrp-server-cloud-bucket/jest.config.js | 23 +- .../package-lock.json | 4 +- infra/uhrp-server-cloud-bucket/package.json | 2 +- .../scripts/sync-secrets.ts | 156 ++-- infra/uhrp-server-cloud-bucket/src/index.ts | 271 ++++--- .../src/routes/list.ts | 6 +- .../src/routes/renew.ts | 89 +- .../src/serviceHealth.test.ts | 42 + .../src/serviceHealth.ts | 24 + .../src/utils/getMetadata.ts | 30 +- .../src/utils/getPriceForFile.ts | 33 +- infra/wab/Dockerfile | 3 + infra/wab/deploy/README.md | 32 + infra/wab/deploy/mysql-deployment.yaml | 39 +- infra/wab/deploy/web-configmap.yaml | 10 + infra/wab/deploy/web-deployment.yaml | 80 +- infra/wab/deploy/web-service.yaml | 4 +- infra/wab/jest.config.ts | 65 +- infra/wab/jest.globalSetup.ts | 16 +- infra/wab/package-lock.json | 4 +- infra/wab/package.json | 2 +- .../src/auth-methods/DevConsoleAuthMethod.ts | 266 +++--- .../wab/src/auth-methods/PersonaAuthMethod.ts | 94 +-- .../wab/src/auth-methods/TwilioAuthMethod.ts | 356 ++++---- infra/wallet-infra/Dockerfile | 3 + .../guides/kube_samples/README.md | 25 + .../guides/kube_samples/mysql.yaml | 82 +- .../guides/kube_samples/wallet.yaml | 103 ++- infra/wallet-infra/package-lock.json | 4 +- infra/wallet-infra/package.json | 2 +- package.json | 8 +- plans/GOVERNANCE.md | 6 + plans/HUMAN_QUESTION.md | 7 +- plans/QUICK_OVERLAYS.md | 7 +- plans/README.md | 17 + plans/RELIABILITY.md | 10 +- plans/SECURITY.md | 5 + plans/TERATESTNET_CHAIN_CONFIG.md | 6 + scripts/service-operations.mjs | 266 ++++++ scripts/service-operations.test.mjs | 12 + 77 files changed, 2747 insertions(+), 1540 deletions(-) create mode 100644 docs/reference/service-operations.md create mode 100644 governance/service-operations.json create mode 100644 infra/overlay-server/deploy/README.md create mode 100644 infra/overlay-server/deploy/app-configmap.yaml create mode 100644 infra/uhrp-server-basic/src/serviceHealth.ts create mode 100644 infra/uhrp-server-basic/test/serviceHealth.test.js create mode 100644 infra/uhrp-server-cloud-bucket/src/serviceHealth.test.ts create mode 100644 infra/uhrp-server-cloud-bucket/src/serviceHealth.ts create mode 100644 infra/wab/deploy/README.md create mode 100644 infra/wab/deploy/web-configmap.yaml create mode 100644 infra/wallet-infra/guides/kube_samples/README.md create mode 100644 plans/README.md create mode 100644 scripts/service-operations.mjs create mode 100644 scripts/service-operations.test.mjs diff --git a/docs/reference/index.md b/docs/reference/index.md index 91e65e73a..ae83ba0f2 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -23,6 +23,7 @@ Quick reference material for API details and protocol indices. - **[Licensing policy](./licensing.md)** — Uniform Open BSV License Version 6 declarations, package files, and drift controls - **[npm package supply chain](./npm-package-supply-chain.md)** — Pack-once candidates, SBOMs, attestations, OIDC publication, and registry verification - **[Release and operations guide](./release-operations.md)** — Scope, preflight, publication, deployment, failure handling, and rollback +- **[Service operations contract](./service-operations.md)** — Generated health, readiness, state, migration, backup, and workload-hardening contracts - **[Repository health controls](./repository-health.md)** — Machine-readable project, package, documentation, test, and exception governance - **[Test quality governance](./test-quality-governance.md)** — Required, manual, live, property, mutation, and conformance test ownership - **[July 2026 modernization release](./release-2026-07-25.md)** — Package versions, compatibility notes, and verification scope diff --git a/docs/reference/service-operations.md b/docs/reference/service-operations.md new file mode 100644 index 000000000..5348be431 --- /dev/null +++ b/docs/reference/service-operations.md @@ -0,0 +1,96 @@ +--- +id: service-operations +title: 'Service Operations Contract' +kind: reference +version: '1.0.0' +last_updated: '2026-07-28' +last_verified: '2026-07-28' +review_cadence_days: 30 +status: stable +tags: [reference, infrastructure, operations, health, recovery] +--- + +# Service Operations Contract + +This page is generated from `governance/service-operations.json`. CI verifies +that all seven released services have a non-root, digest-pinned container with a +real health check and that checked-in application workloads retain startup, +readiness, liveness, resources, seccomp, dropped capabilities, a read-only root +filesystem, and secret indirection. + +## Runtime endpoints + +| Service | Port contract | Liveness | Readiness | Operations | +|---|---|---|---|---| +| `chaintracks-server` | PORT (default 3011; CDN is port + 1) | `/getInfo` | `/getInfo` | [guide](../infrastructure/chaintracks-server.md) | +| `message-box-server` | PORT, then HTTP_PORT (default 8080) | `/health` | `/ready` | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/message-box-server/DEPLOYING.md) | +| `overlay-server` | 8080 | `/health/live` | `/health/ready` | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/overlay-server/deploy/README.md) | +| `uhrp-server-basic` | HTTP_PORT (default 8080) | `/health` | `/ready` | [guide](../infrastructure/uhrp-server-basic.md) | +| `uhrp-server-cloud-bucket` | HTTP_PORT (default 8080) | `/health` | `/ready` | [guide](../infrastructure/uhrp-server-cloud-bucket.md) | +| `wab` | PORT (default 8080) | `/info` | `/info` | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wab/deploy/README.md) | +| `wallet-infra` | HTTP_PORT (default 8081; samples set 8080 without nginx) | `/` | `/` | [guide](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wallet-infra/guides/kube_samples/README.md) | + +Health endpoints are public and non-sensitive. They do not replace protocol +authentication or rate limits. Public services retain wildcard, +credential-free CORS by default; CSP remains a separate document/UI policy. + +## State, migration, and recovery + +### chaintracks-server + +- State: Bulk-header files under BULK_HEADERS_PATH; upstream headers are reproducible. +- Migration/startup: No schema migration. Validate the retained header corpus before rollout. +- Backup/restore: Snapshot BULK_HEADERS_PATH or repopulate it from a verified source CDN. +- Operator guide: [docs/infrastructure/chaintracks-server.md](../infrastructure/chaintracks-server.md) + +### message-box-server + +- State: Knex database plus optional Firebase device registrations. +- Migration/startup: Migrations complete before listen; back up and verify the target schema first. +- Backup/restore: Use the selected database engine's consistent snapshot and restore procedure. +- Operator guide: [infra/message-box-server/DEPLOYING.md](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/message-box-server/DEPLOYING.md) + +### overlay-server + +- State: Knex transaction state and MongoDB lookup-service state. +- Migration/startup: Overlay migrations complete before listen; preserve both stores as one release boundary. +- Backup/restore: Take coordinated MySQL and MongoDB backups before schema or image changes. +- Operator guide: [infra/overlay-server/deploy/README.md](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/overlay-server/deploy/README.md) + +### uhrp-server-basic + +- State: Local files and metadata under the configured public storage directory. +- Migration/startup: No schema migration; preserve file and metadata consistency. +- Backup/restore: Snapshot the complete storage directory and verify hashes before restore. +- Operator guide: [docs/infrastructure/uhrp-server-basic.md](../infrastructure/uhrp-server-basic.md) + +### uhrp-server-cloud-bucket + +- State: Cloud bucket objects and provider metadata. +- Migration/startup: No local schema migration; validate provider configuration before listen. +- Backup/restore: Use provider versioning/replication and verify object hashes and retention policy. +- Operator guide: [docs/infrastructure/uhrp-server-cloud-bucket.md](../infrastructure/uhrp-server-cloud-bucket.md) + +### wab + +- State: Authentication, identity-link, share, deletion-intent, and faucet database tables. +- Migration/startup: Migrations complete before listen; verify rollback compatibility before rollout. +- Backup/restore: Take an encrypted database snapshot and test identity/share recovery without logging secrets. +- Operator guide: [infra/wab/deploy/README.md](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wab/deploy/README.md) + +### wallet-infra + +- State: Wallet Storage database, monitor state, transactions, outputs, baskets, and certificates. +- Migration/startup: Storage migration and availability checks complete before the server starts. +- Backup/restore: Take and verify a consistent database backup before every schema or image change. +- Operator guide: [infra/wallet-infra/guides/kube_samples/README.md](https://github.com/bsv-blockchain/ts-stack/blob/main/infra/wallet-infra/guides/kube_samples/README.md) + +## Change procedure + +1. Change a service, Dockerfile, manifest, or operator guide. +2. Update `governance/service-operations.json` when the operational contract changes. +3. Run `pnpm ops:docs`, then `pnpm ops:check`. +4. Run the affected service tests and the full repository health, container, + documentation, security, and merge gates. +5. Deploy only through a separately authorized release and record the exact image + digest, probe evidence, migration result, backup, and rollback outcome. diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 0110d3894..8ba5094b6 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -75,13 +75,13 @@ the separately released and verified image digest. | Service | Package | Manifest version | Node engine | Runtime targets | Release | Source | | --- | --- | --- | --- | --- | --- | --- | -| BSV Chaintracks Server | `chaintracks-server` | `1.0.10` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/chaintracks-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/chaintracks-server) | -| BSV Message Box Server | `@bsv/messagebox-server` | `1.1.14` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/message-box-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/message-box-server) | -| BSV Overlay Server | `@bsv/overlay-express-examples` | `2.1.18` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/overlay-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/overlay-server) | -| BSV UHRP Basic Server | `@bsv/uhrp-lite` | `0.1.8` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/uhrp-server-basic](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/uhrp-server-basic) | -| BSV UHRP Cloud Bucket Server | `@bsv/uhrp-storage-server` | `0.2.10` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/uhrp-server-cloud-bucket](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/uhrp-server-cloud-bucket) | -| Wallet Authentication Backend | `@bsv/wab-server` | `1.4.11` | `>=24 <25` | node, linux/amd64 | ghcr-and-aws-marketplace-keyless | [infra/wab](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/wab) | -| BSV Wallet Infrastructure | `@bsv/wallet-infra` | `2.0.13` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/wallet-infra](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/wallet-infra) | +| BSV Chaintracks Server | `chaintracks-server` | `1.0.11` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/chaintracks-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/chaintracks-server) | +| BSV Message Box Server | `@bsv/messagebox-server` | `1.1.15` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/message-box-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/message-box-server) | +| BSV Overlay Server | `@bsv/overlay-express-examples` | `2.1.19` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/overlay-server](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/overlay-server) | +| BSV UHRP Basic Server | `@bsv/uhrp-lite` | `0.1.9` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/uhrp-server-basic](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/uhrp-server-basic) | +| BSV UHRP Cloud Bucket Server | `@bsv/uhrp-storage-server` | `0.2.11` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/uhrp-server-cloud-bucket](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/uhrp-server-cloud-bucket) | +| Wallet Authentication Backend | `@bsv/wab-server` | `1.4.12` | `>=24 <25` | node, linux/amd64 | ghcr-and-aws-marketplace-keyless | [infra/wab](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/wab) | +| BSV Wallet Infrastructure | `@bsv/wallet-infra` | `2.0.14` | `>=24 <25` | node, linux/amd64 | ghcr-keyless | [infra/wallet-infra](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/wallet-infra) | ## Governed project and release inventory diff --git a/governance/service-operations.json b/governance/service-operations.json new file mode 100644 index 000000000..566664d89 --- /dev/null +++ b/governance/service-operations.json @@ -0,0 +1,113 @@ +{ + "schemaVersion": 1, + "lastReviewed": "2026-07-28", + "owner": "ts-stack-maintainers", + "manifestRoots": [ + "infra/overlay-server/deploy", + "infra/wab/deploy", + "infra/wallet-infra/guides/kube_samples" + ], + "services": [ + { + "name": "chaintracks-server", + "path": "infra/chaintracks-server", + "port": "PORT (default 3011; CDN is port + 1)", + "livenessPath": "/getInfo", + "readinessPath": "/getInfo", + "state": "Bulk-header files under BULK_HEADERS_PATH; upstream headers are reproducible.", + "migration": "No schema migration. Validate the retained header corpus before rollout.", + "backup": "Snapshot BULK_HEADERS_PATH or repopulate it from a verified source CDN.", + "operatorGuide": "docs/infrastructure/chaintracks-server.md", + "publicProtocol": true + }, + { + "name": "message-box-server", + "path": "infra/message-box-server", + "port": "PORT, then HTTP_PORT (default 8080)", + "livenessPath": "/health", + "readinessPath": "/ready", + "state": "Knex database plus optional Firebase device registrations.", + "migration": "Migrations complete before listen; back up and verify the target schema first.", + "backup": "Use the selected database engine's consistent snapshot and restore procedure.", + "operatorGuide": "infra/message-box-server/DEPLOYING.md", + "publicProtocol": true + }, + { + "name": "overlay-server", + "path": "infra/overlay-server", + "port": "8080", + "livenessPath": "/health/live", + "readinessPath": "/health/ready", + "state": "Knex transaction state and MongoDB lookup-service state.", + "migration": "Overlay migrations complete before listen; preserve both stores as one release boundary.", + "backup": "Take coordinated MySQL and MongoDB backups before schema or image changes.", + "operatorGuide": "infra/overlay-server/deploy/README.md", + "publicProtocol": true + }, + { + "name": "uhrp-server-basic", + "path": "infra/uhrp-server-basic", + "port": "HTTP_PORT (default 8080)", + "livenessPath": "/health", + "readinessPath": "/ready", + "state": "Local files and metadata under the configured public storage directory.", + "migration": "No schema migration; preserve file and metadata consistency.", + "backup": "Snapshot the complete storage directory and verify hashes before restore.", + "operatorGuide": "docs/infrastructure/uhrp-server-basic.md", + "publicProtocol": true + }, + { + "name": "uhrp-server-cloud-bucket", + "path": "infra/uhrp-server-cloud-bucket", + "port": "HTTP_PORT (default 8080)", + "livenessPath": "/health", + "readinessPath": "/ready", + "state": "Cloud bucket objects and provider metadata.", + "migration": "No local schema migration; validate provider configuration before listen.", + "backup": "Use provider versioning/replication and verify object hashes and retention policy.", + "operatorGuide": "docs/infrastructure/uhrp-server-cloud-bucket.md", + "publicProtocol": true + }, + { + "name": "wab", + "path": "infra/wab", + "port": "PORT (default 8080)", + "livenessPath": "/info", + "readinessPath": "/info", + "state": "Authentication, identity-link, share, deletion-intent, and faucet database tables.", + "migration": "Migrations complete before listen; verify rollback compatibility before rollout.", + "backup": "Take an encrypted database snapshot and test identity/share recovery without logging secrets.", + "operatorGuide": "infra/wab/deploy/README.md", + "publicProtocol": true + }, + { + "name": "wallet-infra", + "path": "infra/wallet-infra", + "port": "HTTP_PORT (default 8081; samples set 8080 without nginx)", + "livenessPath": "/", + "readinessPath": "/", + "state": "Wallet Storage database, monitor state, transactions, outputs, baskets, and certificates.", + "migration": "Storage migration and availability checks complete before the server starts.", + "backup": "Take and verify a consistent database backup before every schema or image change.", + "operatorGuide": "infra/wallet-infra/guides/kube_samples/README.md", + "publicProtocol": true + } + ], + "applicationWorkloads": [ + { + "service": "overlay-server", + "manifest": "infra/overlay-server/deploy/app-deployment.yaml", + "container": "overlay" + }, + { + "service": "wab", + "manifest": "infra/wab/deploy/web-deployment.yaml", + "container": "web" + }, + { + "service": "wallet-infra", + "manifest": "infra/wallet-infra/guides/kube_samples/wallet.yaml", + "container": "wallet-infra" + } + ] +} diff --git a/infra/chaintracks-server/Dockerfile b/infra/chaintracks-server/Dockerfile index f72a6884e..a458a64dd 100644 --- a/infra/chaintracks-server/Dockerfile +++ b/infra/chaintracks-server/Dockerfile @@ -58,6 +58,9 @@ USER node # 3012 - CDN Server (bulk headers) EXPOSE 3011 3012 +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD ["node", "-e", "const port=process.env.PORT||'3011';fetch(`http://127.0.0.1:${port}/getInfo`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + # Run the application with the OpenTelemetry bootstrap preloaded so # auto-instrumentation patches modules before app code is imported. CMD ["node", "--require", "./dist/telemetry.js", "dist/server.js"] diff --git a/infra/chaintracks-server/package-lock.json b/infra/chaintracks-server/package-lock.json index 72d2171c1..c3535b93c 100644 --- a/infra/chaintracks-server/package-lock.json +++ b/infra/chaintracks-server/package-lock.json @@ -1,12 +1,12 @@ { "name": "chaintracks-server", - "version": "1.0.10", + "version": "1.0.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "chaintracks-server", - "version": "1.0.10", + "version": "1.0.11", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/wallet-toolbox": "^2.4.4", diff --git a/infra/chaintracks-server/package.json b/infra/chaintracks-server/package.json index 620c9af46..438c92367 100644 --- a/infra/chaintracks-server/package.json +++ b/infra/chaintracks-server/package.json @@ -1,6 +1,6 @@ { "name": "chaintracks-server", - "version": "1.0.10", + "version": "1.0.11", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/infra/message-box-server/Dockerfile b/infra/message-box-server/Dockerfile index 381ed42a6..2a64b7525 100644 --- a/infra/message-box-server/Dockerfile +++ b/infra/message-box-server/Dockerfile @@ -52,6 +52,9 @@ EXPOSE 8080 USER node +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD ["node", "-e", "const port=process.env.PORT||process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/ready`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + # Start the app with the OpenTelemetry bootstrap preloaded (ESM --import) so # auto-instrumentation patches modules before they are imported. CMD [ "node", "--import", "./out/src/telemetry.js", "out/src/index.js" ] diff --git a/infra/message-box-server/package-lock.json b/infra/message-box-server/package-lock.json index 1bb4e7d0b..50fdf8972 100644 --- a/infra/message-box-server/package-lock.json +++ b/infra/message-box-server/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bsv/messagebox-server", - "version": "1.1.14", + "version": "1.1.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bsv/messagebox-server", - "version": "1.1.14", + "version": "1.1.15", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/auth-express-middleware": "^2.1.2", diff --git a/infra/message-box-server/package.json b/infra/message-box-server/package.json index 8e4a18f41..c14a538b6 100644 --- a/infra/message-box-server/package.json +++ b/infra/message-box-server/package.json @@ -1,7 +1,7 @@ { "name": "@bsv/messagebox-server", "private": true, - "version": "1.1.14", + "version": "1.1.15", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/infra/message-box-server/src/config/firebase.ts b/infra/message-box-server/src/config/firebase.ts index b4989678b..8bb30ec9a 100644 --- a/infra/message-box-server/src/config/firebase.ts +++ b/infra/message-box-server/src/config/firebase.ts @@ -4,11 +4,13 @@ import { getApp, cert, applicationDefault, - type App + type App, + type Credential } from 'firebase-admin/app' import { getMessaging, type Messaging, type Message } from 'firebase-admin/messaging' import { getFirestore, type Firestore } from 'firebase-admin/firestore' import * as path from 'node:path' +import { readFileSync } from 'node:fs' import dotenv from 'dotenv' import { log } from '../utils/logger.js' @@ -16,6 +18,60 @@ dotenv.config() let firebaseApp: App | null = null +function credentialFromJson(rawCredential: string, source: string): Credential { + try { + const serviceAccount: unknown = JSON.parse(rawCredential) + if (serviceAccount == null || typeof serviceAccount !== 'object') { + throw new TypeError('Parsed service account is not a valid object') + } + const values = serviceAccount as Record + for (const field of ['private_key', 'client_email', 'project_id']) { + if (typeof values[field] !== 'string' || values[field] === '') { + throw new TypeError( + 'Service account missing required fields (private_key, client_email, project_id)' + ) + } + } + return cert({ + privateKey: values.private_key as string, + clientEmail: values.client_email as string, + projectId: values.project_id as string + }) + } catch (error) { + throw new TypeError( + `Failed to parse Firebase service account from ${source}: ${error instanceof Error ? error.message : 'Invalid JSON'}`, + { cause: error } + ) + } +} + +function resolveFirebaseCredential(): Credential { + const serviceAccountJson = process.env.FIREBASE_SERVICE_ACCOUNT_JSON + if (serviceAccountJson != null && serviceAccountJson !== '') { + log.info( + { operation: 'firebase.init', credential_source: 'env' }, + 'Using Firebase service account from environment variable' + ) + return credentialFromJson(serviceAccountJson, 'FIREBASE_SERVICE_ACCOUNT_JSON') + } + + const serviceAccountPath = process.env.FIREBASE_SERVICE_ACCOUNT_PATH + if (serviceAccountPath != null && serviceAccountPath !== '') { + log.info( + { operation: 'firebase.init', credential_source: 'file' }, + 'Using Firebase service account key file' + ) + const absolutePath = path.resolve(process.cwd(), serviceAccountPath) + return credentialFromJson(readFileSync(absolutePath, 'utf8'), absolutePath) + } + + log.info( + { operation: 'firebase.init', credential_source: 'default' }, + 'Using Firebase default credentials' + ) + return applicationDefault() +} + /** * Initialize Firebase Admin SDK. * Returns null (and logs a warning) when ENABLE_FIREBASE is not 'true', @@ -38,86 +94,13 @@ export function initializeFirebase(): App | null { } try { - const serviceAccountJson = process.env.FIREBASE_SERVICE_ACCOUNT_JSON - const serviceAccountPath = process.env.FIREBASE_SERVICE_ACCOUNT_PATH const projectId = process.env.FIREBASE_PROJECT_ID if (projectId == null || projectId === '') { - throw new Error('FIREBASE_PROJECT_ID environment variable is required') + throw new TypeError('FIREBASE_PROJECT_ID environment variable is required') } - let firebaseCredential: any // Will be assigned based on auth method - - if (serviceAccountJson != null && serviceAccountJson !== '') { - log.info( - { operation: 'firebase.init', credential_source: 'env' }, - 'Using Firebase service account from environment variable' - ) - try { - log.debug( - { operation: 'firebase.init', service_account_json_length: serviceAccountJson.length }, - 'Service account JSON length' - ) - - // Debug credential functions - log.debug( - { operation: 'firebase.init', cert_function_type: typeof cert }, - 'cert function type' - ) - log.debug( - { - operation: 'firebase.init', - application_default_function_type: typeof applicationDefault - }, - 'applicationDefault function type' - ) - - const serviceAccount = JSON.parse(serviceAccountJson) - log.debug( - { operation: 'firebase.init', service_account_keys: Object.keys(serviceAccount ?? {}) }, - 'Parsed service account keys' - ) - - if (serviceAccount == null || typeof serviceAccount !== 'object') { - throw new Error('Parsed service account is not a valid object') - } - - if ( - serviceAccount.private_key == null || - serviceAccount.client_email == null || - serviceAccount.project_id == null - ) { - throw new Error( - 'Service account missing required fields (private_key, client_email, project_id)' - ) - } - - firebaseCredential = cert(serviceAccount) - log.info({ operation: 'firebase.init' }, 'Firebase credential created successfully') - } catch (parseError) { - log.error( - { operation: 'firebase.init', outcome: 'error', err: parseError }, - 'Firebase service account parsing failed' - ) - throw new Error( - `Failed to parse FIREBASE_SERVICE_ACCOUNT_JSON: ${parseError instanceof Error ? parseError.message : 'Invalid JSON'}` - ) - } - } else if (serviceAccountPath != null && serviceAccountPath !== '') { - log.info( - { operation: 'firebase.init', credential_source: 'file' }, - 'Using Firebase service account key file' - ) - const absolutePath = path.resolve(process.cwd(), serviceAccountPath) - // eslint-disable-next-line @typescript-eslint/no-var-requires - firebaseCredential = cert(require(absolutePath)) - } else { - log.info( - { operation: 'firebase.init', credential_source: 'default' }, - 'Using Firebase default credentials' - ) - firebaseCredential = applicationDefault() - } + const firebaseCredential = resolveFirebaseCredential() // Check if Firebase app is already initialized if (getApps().length === 0) { diff --git a/infra/message-box-server/src/migrations/2025-01-31-001-notification-permissions.ts b/infra/message-box-server/src/migrations/2025-01-31-001-notification-permissions.ts index 150619b22..028570537 100644 --- a/infra/message-box-server/src/migrations/2025-01-31-001-notification-permissions.ts +++ b/infra/message-box-server/src/migrations/2025-01-31-001-notification-permissions.ts @@ -31,7 +31,7 @@ export async function up(knex: Knex): Promise { // Insert default server fees for different message box types const existingFees = await knex('server_fees').select('message_box') - const existingMessageBoxes = existingFees.map(fee => fee.message_box) + const existingMessageBoxes = new Set(existingFees.map(fee => fee.message_box)) const defaultFees = [ { @@ -55,7 +55,7 @@ export async function up(knex: Knex): Promise { ] // Only insert fees for message boxes that don't already exist - const feesToInsert = defaultFees.filter(fee => !existingMessageBoxes.includes(fee.message_box)) + const feesToInsert = defaultFees.filter(fee => !existingMessageBoxes.has(fee.message_box)) if (feesToInsert.length > 0) { await knex('server_fees').insert(feesToInsert) diff --git a/infra/message-box-server/src/routes/permissions/setPermission.ts b/infra/message-box-server/src/routes/permissions/setPermission.ts index 05852dc54..ccc7ff89c 100644 --- a/infra/message-box-server/src/routes/permissions/setPermission.ts +++ b/infra/message-box-server/src/routes/permissions/setPermission.ts @@ -15,6 +15,32 @@ export interface SetPermissionRequestType extends AuthRequest { } } +const validRecipientFee = (recipientFee: number): boolean => + Number.isSafeInteger(recipientFee) && recipientFee >= -1 && recipientFee <= MAX_RECIPIENT_FEE + +const validMessageBox = (messageBox: unknown): messageBox is string => + typeof messageBox === 'string' && + messageBox.trim() !== '' && + Buffer.byteLength(messageBox.trim(), 'utf8') <= MAX_PERMISSION_MESSAGE_BOX_BYTES + +function permissionDescription( + sender: string | undefined, + messageBox: string, + recipientFee: number +): string { + const isBoxWide = sender == null + const senderText = sender ?? 'all senders' + const actionText = isBoxWide ? 'Box-wide default for' : 'Messages from' + + if (recipientFee === -1) { + return `${actionText} ${senderText} to ${messageBox} ${isBoxWide ? 'is' : 'are'} now blocked.` + } + if (recipientFee === 0) { + return `${actionText} ${senderText} to ${messageBox} ${isBoxWide ? 'is' : 'are'} now always allowed.` + } + return `${actionText} ${senderText} to ${messageBox} now require${isBoxWide ? 's' : ''} ${recipientFee} satoshis.` +} + /** * @swagger * /permissions/set: @@ -98,11 +124,7 @@ export default { } // Validate recipientFee value - if ( - !Number.isSafeInteger(recipientFee) || - recipientFee < -1 || - recipientFee > MAX_RECIPIENT_FEE - ) { + if (!validRecipientFee(recipientFee)) { Logger.log('[DEBUG] Invalid recipientFee value - must be integer') return res.status(400).json({ status: 'error', @@ -112,11 +134,7 @@ export default { } // Validate messageBox value - if ( - typeof messageBox !== 'string' || - messageBox.trim() === '' || - Buffer.byteLength(messageBox.trim(), 'utf8') > MAX_PERMISSION_MESSAGE_BOX_BYTES - ) { + if (!validMessageBox(messageBox)) { Logger.log('[DEBUG] Invalid messageBox value') return res.status(400).json({ status: 'error', @@ -141,26 +159,13 @@ export default { }) } - const isBoxWide = sender == null Logger.log( `[DEBUG] Successfully updated message permission: ${sender ?? 'BOX-WIDE'} -> ${recipient} (${messageBox}), fee: ${recipientFee}` ) - let description: string - const senderText = isBoxWide ? 'all senders' : sender - const actionText = isBoxWide ? 'Box-wide default for' : 'Messages from' - - if (recipientFee === -1) { - description = `${actionText} ${senderText} to ${messageBox} ${isBoxWide ? 'is' : 'are'} now blocked.` - } else if (recipientFee === 0) { - description = `${actionText} ${senderText} to ${messageBox} ${isBoxWide ? 'is' : 'are'} now always allowed.` - } else { - description = `${actionText} ${senderText} to ${messageBox} now require${isBoxWide ? 's' : ''} ${recipientFee} satoshis.` - } - return res.status(200).json({ status: 'success', - description + description: permissionDescription(sender, messageBox, recipientFee) }) } catch (error) { Logger.error('[ERROR] Internal Server Error in set permission:', error) diff --git a/infra/overlay-server/Dockerfile b/infra/overlay-server/Dockerfile index 0c81e8b83..4da19d5f8 100644 --- a/infra/overlay-server/Dockerfile +++ b/infra/overlay-server/Dockerfile @@ -27,4 +27,7 @@ USER node EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:8080/health/ready').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "--import", "./dist/telemetry.js", "dist/index.js"] diff --git a/infra/overlay-server/deploy/README.md b/infra/overlay-server/deploy/README.md new file mode 100644 index 000000000..a7863cced --- /dev/null +++ b/infra/overlay-server/deploy/README.md @@ -0,0 +1,30 @@ +# Overlay Kubernetes deployment + +These manifests are secure starting points, not a live environment definition. Pin +every application image to the release tag and digest verified by the repository +release workflow. The checked-in digest is an example release and is not updated by +an application rollout. + +Before applying the manifests, create two secret objects through the operator's +secret manager or deployment system: + +- `overlay-secrets`: `knex-url`, `mongo-url`, `server-private-key`, + `arc-api-key`, and `admin-token`. +- `overlay-database-secrets`: `mongo-root-user`, `mongo-root-password`, + `mysql-database`, `mysql-user`, `mysql-password`, and + `mysql-root-password`. + +Do not commit Secret manifests or literal credentials. Update `overlay-config` +with the public node name, hosting URL, wallet-storage URL, network, and GASP +choice. The API remains public and credential-free wildcard CORS by default; +origin allowlists are an explicit operator option, not a deployment prerequisite. + +The application image runs directly with Node on port 8080. Startup and liveness +use `/health/live`; readiness uses `/health/ready`, which includes configured +database and engine checks. The application filesystem is read-only and the +container runs without Linux capabilities. Database pods retain writable PVCs. + +Back up both databases before changing a schema or image. Roll out the databases +separately from the application, wait for their probes, then update the +application digest. Roll back by restoring the prior digest; restore data only +under the database engine's documented recovery procedure. diff --git a/infra/overlay-server/deploy/app-configmap.yaml b/infra/overlay-server/deploy/app-configmap.yaml new file mode 100644 index 000000000..354154a1d --- /dev/null +++ b/infra/overlay-server/deploy/app-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: overlay-config +data: + node-name: replace-with-node-name + hosting-url: https://overlay.example.com + wallet-storage-url: https://storage.example.com + network: main + gasp-enabled: 'true' diff --git a/infra/overlay-server/deploy/app-deployment.yaml b/infra/overlay-server/deploy/app-deployment.yaml index 599ffb1e0..9769a7495 100644 --- a/infra/overlay-server/deploy/app-deployment.yaml +++ b/infra/overlay-server/deploy/app-deployment.yaml @@ -21,38 +21,105 @@ spec: io.kompose.service: app spec: automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 containers: - - args: - - npm - - run - - start - env: + - env: + - name: NODE_NAME + valueFrom: + configMapKeyRef: + name: overlay-config + key: node-name + - name: HOSTING_URL + valueFrom: + configMapKeyRef: + name: overlay-config + key: hosting-url + - name: WALLET_STORAGE_URL + valueFrom: + configMapKeyRef: + name: overlay-config + key: wallet-storage-url + - name: NETWORK + valueFrom: + configMapKeyRef: + name: overlay-config + key: network + - name: GASP_ENABLED + valueFrom: + configMapKeyRef: + name: overlay-config + key: gasp-enabled - name: KNEX_URL valueFrom: secretKeyRef: name: overlay-secrets key: knex-url - name: MONGO_URL - value: mongodb://root:example@mongodb:27017 + valueFrom: + secretKeyRef: + name: overlay-secrets + key: mongo-url - name: SERVER_PRIVATE_KEY - value: "" + valueFrom: + secretKeyRef: + name: overlay-secrets + key: server-private-key - name: ARC_API_KEY - value: "" - - name: HOSTING_URL - value: "" + valueFrom: + secretKeyRef: + name: overlay-secrets + key: arc-api-key - name: ADMIN_TOKEN - value: "" + valueFrom: + secretKeyRef: + name: overlay-secrets + key: admin-token image: ghcr.io/bsv-blockchain/overlay-server:v2.1.18@sha256:11e6c1ba404fec9c4ad300e5c3e0e21dce3f1ceddbba733c7ecce57801645fa9 + imagePullPolicy: IfNotPresent name: overlay ports: - - containerPort: 3000 + - containerPort: 8080 + name: http protocol: TCP + startupProbe: + httpGet: + path: /health/live + port: http + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /health/ready + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health/live + port: http + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 resources: requests: cpu: 100m memory: 256Mi ephemeral-storage: 1Gi limits: + cpu: '1' memory: 512Mi ephemeral-storage: 2Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 restartPolicy: Always diff --git a/infra/overlay-server/deploy/app-service.yaml b/infra/overlay-server/deploy/app-service.yaml index 573484a5d..0db8275f4 100644 --- a/infra/overlay-server/deploy/app-service.yaml +++ b/infra/overlay-server/deploy/app-service.yaml @@ -9,8 +9,8 @@ metadata: name: app spec: ports: - - name: "3000" - port: 3000 - targetPort: 3000 + - name: http + port: 8080 + targetPort: http selector: io.kompose.service: app diff --git a/infra/overlay-server/deploy/janitor-cronjob.yaml b/infra/overlay-server/deploy/janitor-cronjob.yaml index abc863ce0..cb40b3662 100644 --- a/infra/overlay-server/deploy/janitor-cronjob.yaml +++ b/infra/overlay-server/deploy/janitor-cronjob.yaml @@ -8,7 +8,7 @@ metadata: io.kompose.service: janitor name: janitor spec: - schedule: "0 0 * * *" + schedule: '0 0 * * *' jobTemplate: spec: template: @@ -16,15 +16,40 @@ spec: labels: io.kompose.service: janitor spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: janitor image: curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13 env: - name: ADMIN_TOKEN - value: "" + valueFrom: + secretKeyRef: + name: overlay-secrets + key: admin-token command: - /bin/sh - -c - | - curl -X POST -H "Authorization: Bearer ${ADMIN_TOKEN}" http://app:3000/admin/janitor + curl --fail --show-error --silent -X POST \ + -H "Authorization: Bearer ${ADMIN_TOKEN}" \ + http://app:8080/admin/janitor + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 100 + runAsGroup: 101 restartPolicy: OnFailure diff --git a/infra/overlay-server/deploy/mongodb-deployment.yaml b/infra/overlay-server/deploy/mongodb-deployment.yaml index c80291b89..f289f56b0 100644 --- a/infra/overlay-server/deploy/mongodb-deployment.yaml +++ b/infra/overlay-server/deploy/mongodb-deployment.yaml @@ -26,26 +26,52 @@ spec: containers: - env: - name: MONGO_INITDB_ROOT_PASSWORD - value: example + valueFrom: + secretKeyRef: + name: overlay-database-secrets + key: mongo-root-password - name: MONGO_INITDB_ROOT_USERNAME - value: root + valueFrom: + secretKeyRef: + name: overlay-database-secrets + key: mongo-root-user image: mongo:6.0.28@sha256:8b6d8f5bbedb25cb73517b65cf99f13aeb75ad5b157a56c479287a840bbad3ac name: mongo ports: - containerPort: 27017 + name: mongodb protocol: TCP + startupProbe: + exec: + command: + - mongosh + - --quiet + - --eval + - db.adminCommand('ping') + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + tcpSocket: + port: mongodb + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: mongodb + periodSeconds: 30 resources: requests: cpu: 100m memory: 256Mi ephemeral-storage: 1Gi limits: + cpu: '1' memory: 512Mi ephemeral-storage: 2Gi volumeMounts: - mountPath: /data/db name: mongo-data restartPolicy: Always + terminationGracePeriodSeconds: 60 volumes: - name: mongo-data persistentVolumeClaim: diff --git a/infra/overlay-server/deploy/mysql-deployment.yaml b/infra/overlay-server/deploy/mysql-deployment.yaml index b119c796f..bb6336629 100644 --- a/infra/overlay-server/deploy/mysql-deployment.yaml +++ b/infra/overlay-server/deploy/mysql-deployment.yaml @@ -26,30 +26,62 @@ spec: containers: - env: - name: MYSQL_DATABASE - value: appdb + valueFrom: + secretKeyRef: + name: overlay-database-secrets + key: mysql-database - name: MYSQL_PASSWORD - value: apppass + valueFrom: + secretKeyRef: + name: overlay-database-secrets + key: mysql-password - name: MYSQL_ROOT_PASSWORD - value: example + valueFrom: + secretKeyRef: + name: overlay-database-secrets + key: mysql-root-password - name: MYSQL_USER - value: appuser + valueFrom: + secretKeyRef: + name: overlay-database-secrets + key: mysql-user image: mysql:8.4.10@sha256:8dbcf531a03aade657e181b9cf2f1d1803ce621a1d55610cb44cb531ab7d7db6 name: mysql ports: - containerPort: 3306 + name: mysql protocol: TCP + startupProbe: + exec: + command: + - mysqladmin + - ping + - --host=127.0.0.1 + - --silent + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + tcpSocket: + port: mysql + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: mysql + periodSeconds: 30 resources: requests: cpu: 100m memory: 512Mi ephemeral-storage: 1Gi limits: + cpu: '1' memory: 1Gi ephemeral-storage: 2Gi volumeMounts: - mountPath: /var/lib/mysql name: mysql-data restartPolicy: Always + terminationGracePeriodSeconds: 60 volumes: - name: mysql-data persistentVolumeClaim: diff --git a/infra/overlay-server/package-lock.json b/infra/overlay-server/package-lock.json index 095c5f830..869ecee6d 100644 --- a/infra/overlay-server/package-lock.json +++ b/infra/overlay-server/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bsv/overlay-express-examples", - "version": "2.1.18", + "version": "2.1.19", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bsv/overlay-express-examples", - "version": "2.1.18", + "version": "2.1.19", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/overlay": "^2.2.1", diff --git a/infra/overlay-server/package.json b/infra/overlay-server/package.json index 573b1171e..de2a4a6f1 100644 --- a/infra/overlay-server/package.json +++ b/infra/overlay-server/package.json @@ -1,7 +1,7 @@ { "name": "@bsv/overlay-express-examples", "private": true, - "version": "2.1.18", + "version": "2.1.19", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/infra/overlay-server/src/index.ts b/infra/overlay-server/src/index.ts index d65696c9b..41b486508 100644 --- a/infra/overlay-server/src/index.ts +++ b/infra/overlay-server/src/index.ts @@ -1,46 +1,46 @@ import { WalletAdvertiser } from '@bsv/overlay-discovery-services' import OverlayExpress from '@bsv/overlay-express' import { - ProtoMapTopicManager, - createProtoMapLookupService, - CertMapTopicManager, - createCertMapLookupService, - BasketMapTopicManager, - createBasketMapLookupService, - UHRPTopicManager, - createUHRPLookupService, - IdentityTopicManager, - createIdentityLookupService, - MessageBoxTopicManager, - createMessageBoxLookupService, - UMPTopicManager, - createUMPLookupService, - HelloWorldTopicManager, - createHelloWorldLookupService, - SlackThreadsTopicManager, - createSlackThreadsLookupService, - DesktopIntegrityTopicManager, - createDesktopIntegrityLookupService, - FractionalizeTopicManager, - createFractionalizeLookupService, - SupplyChainTopicManager, - createSupplyChainLookupService, - MonsterBattleTopicManager, - createMonsterBattleLookupService, - AnyTopicManager, - createAnyLookupService, - AppsTopicManager, - createAppsLookupService, - DIDTopicManager, - createDIDLookupService, - WalletConfigTopicManager, - createWalletConfigLookupService, - TokenDemoTopicManager, - createTokenDemoLookupService, - MandalaTopicManager, - MandalaStorageManager, - createMandalaLookupService, - InMemoryScreeningProvider, + ProtoMapTopicManager, + createProtoMapLookupService, + CertMapTopicManager, + createCertMapLookupService, + BasketMapTopicManager, + createBasketMapLookupService, + UHRPTopicManager, + createUHRPLookupService, + IdentityTopicManager, + createIdentityLookupService, + MessageBoxTopicManager, + createMessageBoxLookupService, + UMPTopicManager, + createUMPLookupService, + HelloWorldTopicManager, + createHelloWorldLookupService, + SlackThreadsTopicManager, + createSlackThreadsLookupService, + DesktopIntegrityTopicManager, + createDesktopIntegrityLookupService, + FractionalizeTopicManager, + createFractionalizeLookupService, + SupplyChainTopicManager, + createSupplyChainLookupService, + MonsterBattleTopicManager, + createMonsterBattleLookupService, + AnyTopicManager, + createAnyLookupService, + AppsTopicManager, + createAppsLookupService, + DIDTopicManager, + createDIDLookupService, + WalletConfigTopicManager, + createWalletConfigLookupService, + TokenDemoTopicManager, + createTokenDemoLookupService, + MandalaTopicManager, + MandalaStorageManager, + createMandalaLookupService, + InMemoryScreeningProvider } from '@bsv/overlay-topics' import { PrivateKey, ProtoWallet, WalletInterface } from '@bsv/sdk' @@ -54,371 +54,402 @@ const tracer = trace.getTracer(packageJson.name, packageJson.version) // Reads a required environment variable, failing fast with a clear message if it is missing. const requireEnv = (name: string): string => { - const value = process.env[name] - if (value === undefined || value === '') { - throw new Error(`Missing required environment variable: ${name}`) - } - return value + const value = process.env[name] + if (value === undefined || value === '') { + throw new TypeError(`Missing required environment variable: ${name}`) + } + return value } const optionalEnv = (name: string): string | undefined => { - const value = process.env[name] - return value === undefined || value === '' ? undefined : value + const value = process.env[name] + return value === undefined || value === '' ? undefined : value } const optionalSecretEnv = (name: string, minimumLength: number): string | undefined => { - const value = optionalEnv(name) - if (value !== undefined && value.length < minimumLength) { - throw new Error(`${name} must contain at least ${minimumLength} characters`) - } - return value + const value = optionalEnv(name) + if (value !== undefined && value.length < minimumLength) { + throw new TypeError(`${name} must contain at least ${minimumLength} characters`) + } + return value } const boolEnv = (name: string, defaultValue: boolean): boolean => { - const value = optionalEnv(name) - if (value === undefined) return defaultValue - return value === 'true' || value === '1' || value === 'yes' + const value = optionalEnv(name) + if (value === undefined) return defaultValue + return value === 'true' || value === '1' || value === 'yes' } const numberEnv = (name: string): number | undefined => { - const value = optionalEnv(name) - if (value === undefined) return undefined - const parsed = Number(value) - if (!Number.isFinite(parsed)) { - throw new Error(`${name} must be a finite number, got: ${value}`) - } - return parsed + const value = optionalEnv(name) + if (value === undefined) return undefined + const parsed = Number(value) + if (!Number.isFinite(parsed)) { + throw new TypeError(`${name} must be a finite number, got: ${value}`) + } + return parsed } -const overlayLogArgs = (args: unknown[]): { message: string, fields: Record } => { - const [first, ...rest] = args - if (typeof first === 'string') { - return { - message: first, - fields: rest.length > 0 ? { operation: 'overlay', args: rest } : { operation: 'overlay' } - } - } +const networkEnv = (): 'main' | 'test' => { + const network = requireEnv('NETWORK') + if (network !== 'main' && network !== 'test') { + throw new TypeError(`NETWORK must be "main" or "test", got: ${network}`) + } + return network +} + +const requirePropagationProvider = ( + arcApiKey: string | undefined, + arcadeUrl: string | undefined +): void => { + if (arcApiKey === undefined && arcadeUrl === undefined) { + throw new TypeError( + 'Configure at least one transaction propagation provider: ARC_API_KEY or ARCADE_URL' + ) + } +} + +const overlayLogArgs = (args: unknown[]): { message: string; fields: Record } => { + const [first, ...rest] = args + if (typeof first === 'string') { return { - message: 'overlay log', - fields: { - operation: 'overlay', - event: first, - args: rest - } + message: first, + fields: rest.length > 0 ? { operation: 'overlay', args: rest } : { operation: 'overlay' } } + } + return { + message: 'overlay log', + fields: { + operation: 'overlay', + event: first, + args: rest + } + } } const overlayLogger = { - log: (...args: unknown[]) => { - const entry = overlayLogArgs(args) - log.info(entry.fields, entry.message) - }, - info: (...args: unknown[]) => { - const entry = overlayLogArgs(args) - log.info(entry.fields, entry.message) - }, - warn: (...args: unknown[]) => { - const entry = overlayLogArgs(args) - log.warn(entry.fields, entry.message) - }, - error: (...args: unknown[]) => { - const entry = overlayLogArgs(args) - log.error(entry.fields, entry.message) - }, - debug: (...args: unknown[]) => { - const entry = overlayLogArgs(args) - log.debug(entry.fields, entry.message) - } + log: (...args: unknown[]) => { + const entry = overlayLogArgs(args) + log.info(entry.fields, entry.message) + }, + info: (...args: unknown[]) => { + const entry = overlayLogArgs(args) + log.info(entry.fields, entry.message) + }, + warn: (...args: unknown[]) => { + const entry = overlayLogArgs(args) + log.warn(entry.fields, entry.message) + }, + error: (...args: unknown[]) => { + const entry = overlayLogArgs(args) + log.error(entry.fields, entry.message) + }, + debug: (...args: unknown[]) => { + const entry = overlayLogArgs(args) + log.debug(entry.fields, entry.message) + } } type OverlayProviderConfigMethods = OverlayExpress & { - configureArcade?: (url: string, config?: { - apiKey?: string - deploymentId?: string - chaintracksApiPrefix?: string - }) => void - configureChaintracks?: (url: string, config?: { - apiPrefix?: string - reorgStream?: boolean - scanDepth?: number - }) => void - configureUnprovenMaintenance?: (config: { intervalMs?: number, thresholdBlocks?: number }) => void + configureArcade?: ( + url: string, + config?: { + apiKey?: string + deploymentId?: string + chaintracksApiPrefix?: string + } + ) => void + configureChaintracks?: ( + url: string, + config?: { + apiPrefix?: string + reorgStream?: boolean + scanDepth?: number + } + ) => void + configureUnprovenMaintenance?: (config: { intervalMs?: number; thresholdBlocks?: number }) => void } // Hi there! Let's configure Overlay Express! const main = async () => { - // Validate required configuration up front so misconfiguration fails fast. - const NODE_NAME = requireEnv('NODE_NAME') - const SERVER_PRIVATE_KEY = requireEnv('SERVER_PRIVATE_KEY') - const HOSTING_URL = requireEnv('HOSTING_URL') - const WALLET_STORAGE_URL = requireEnv('WALLET_STORAGE_URL') - const ARC_API_KEY = optionalEnv('ARC_API_KEY') - const ARC_CALLBACK_TOKEN = optionalEnv('ARC_CALLBACK_TOKEN') - const ARCADE_URL = optionalEnv('ARCADE_URL') - const ARCADE_API_KEY = optionalEnv('ARCADE_API_KEY') - const ARCADE_DEPLOYMENT_ID = optionalEnv('ARCADE_DEPLOYMENT_ID') - const CHAINTRACKS_URL = optionalEnv('CHAINTRACKS_URL') - const CHAINTRACKS_API_PREFIX = optionalEnv('CHAINTRACKS_API_PREFIX') ?? '/chaintracks/v2' - const KNEX_URL = requireEnv('KNEX_URL') - const MONGO_URL = requireEnv('MONGO_URL') - const ADMIN_TOKEN = optionalSecretEnv('ADMIN_TOKEN', 32) // random token generated if unset - - const NETWORK = requireEnv('NETWORK') - if (NETWORK !== 'main' && NETWORK !== 'test') { - throw new Error(`NETWORK must be "main" or "test", got: ${NETWORK}`) - } - if (ARC_API_KEY === undefined && ARCADE_URL === undefined) { - throw new Error('Configure at least one transaction propagation provider: ARC_API_KEY or ARCADE_URL') + // Validate required configuration up front so misconfiguration fails fast. + const NODE_NAME = requireEnv('NODE_NAME') + const SERVER_PRIVATE_KEY = requireEnv('SERVER_PRIVATE_KEY') + const HOSTING_URL = requireEnv('HOSTING_URL') + const WALLET_STORAGE_URL = requireEnv('WALLET_STORAGE_URL') + const ARC_API_KEY = optionalEnv('ARC_API_KEY') + const ARC_CALLBACK_TOKEN = optionalEnv('ARC_CALLBACK_TOKEN') + const ARCADE_URL = optionalEnv('ARCADE_URL') + const ARCADE_API_KEY = optionalEnv('ARCADE_API_KEY') + const ARCADE_DEPLOYMENT_ID = optionalEnv('ARCADE_DEPLOYMENT_ID') + const CHAINTRACKS_URL = optionalEnv('CHAINTRACKS_URL') + const CHAINTRACKS_API_PREFIX = optionalEnv('CHAINTRACKS_API_PREFIX') ?? '/chaintracks/v2' + const KNEX_URL = requireEnv('KNEX_URL') + const MONGO_URL = requireEnv('MONGO_URL') + const ADMIN_TOKEN = optionalSecretEnv('ADMIN_TOKEN', 32) // random token generated if unset + + const NETWORK = networkEnv() + requirePropagationProvider(ARC_API_KEY, ARCADE_URL) + + // We'll make a new server for our overlay node. + const server = new OverlayExpress( + // Name your overlay node with a one-word lowercase string + NODE_NAME, + + // Provide the private key that gives your node its identity + SERVER_PRIVATE_KEY, + + // Provide the HTTPS URL where your node is available on the internet + HOSTING_URL, + + // Provide an adminToken to enable the admin API + ADMIN_TOKEN + ) + const providerServer = server as OverlayProviderConfigMethods + server.configureLogger(overlayLogger as unknown as typeof console) + + const wa = new WalletAdvertiser(NETWORK, SERVER_PRIVATE_KEY, WALLET_STORAGE_URL, HOSTING_URL) + + await wa.init() + + server.configureEngineParams({ + advertiser: wa, + throwOnBroadcastFailure: boolEnv('THROW_ON_BROADCAST_FAIL', true) + }) + + server.configureNetwork(NETWORK) + + if (ARC_CALLBACK_TOKEN !== undefined) { + server.configureArcCallbackToken(ARC_CALLBACK_TOKEN) + } + + if (ARCADE_URL !== undefined) { + if (typeof providerServer.configureArcade !== 'function') { + throw new TypeError( + 'ARCADE_URL requires an @bsv/overlay-express version with configureArcade support' + ) } - - // We'll make a new server for our overlay node. - const server = new OverlayExpress( - - // Name your overlay node with a one-word lowercase string - NODE_NAME, - - // Provide the private key that gives your node its identity - SERVER_PRIVATE_KEY, - - // Provide the HTTPS URL where your node is available on the internet - HOSTING_URL, - - // Provide an adminToken to enable the admin API - ADMIN_TOKEN - ) - const providerServer = server as OverlayProviderConfigMethods - server.configureLogger(overlayLogger as unknown as typeof console) - - const wa = new WalletAdvertiser( - NETWORK, - SERVER_PRIVATE_KEY, - WALLET_STORAGE_URL, - HOSTING_URL - ) - - await wa.init() - - server.configureEngineParams({ - advertiser: wa, - throwOnBroadcastFailure: boolEnv('THROW_ON_BROADCAST_FAIL', true) + providerServer.configureArcade(ARCADE_URL, { + apiKey: ARCADE_API_KEY, + deploymentId: ARCADE_DEPLOYMENT_ID, + chaintracksApiPrefix: CHAINTRACKS_API_PREFIX }) - - server.configureNetwork(NETWORK) - - if (ARC_CALLBACK_TOKEN !== undefined) { - server.configureArcCallbackToken(ARC_CALLBACK_TOKEN) - } - - if (ARCADE_URL !== undefined) { - if (typeof providerServer.configureArcade !== 'function') { - throw new Error('ARCADE_URL requires an @bsv/overlay-express version with configureArcade support') - } - providerServer.configureArcade(ARCADE_URL, { - apiKey: ARCADE_API_KEY, - deploymentId: ARCADE_DEPLOYMENT_ID, - chaintracksApiPrefix: CHAINTRACKS_API_PREFIX - }) - } - - if (ARC_API_KEY !== undefined) { - server.configureArcApiKey(ARC_API_KEY) - } - - const chaintracksUrl = CHAINTRACKS_URL ?? (boolEnv('USE_ARCADE_CHAINTRACKS', ARCADE_URL !== undefined) ? ARCADE_URL : undefined) - if (chaintracksUrl !== undefined) { - if (typeof providerServer.configureChaintracks !== 'function') { - throw new Error('CHAINTRACKS_URL/USE_ARCADE_CHAINTRACKS requires an @bsv/overlay-express version with configureChaintracks support') - } - providerServer.configureChaintracks(chaintracksUrl, { - apiPrefix: CHAINTRACKS_API_PREFIX, - reorgStream: boolEnv('BASM_REORG_STREAM_ENABLED', true), - scanDepth: numberEnv('BASM_REORG_SCAN_DEPTH') - }) + } + + if (ARC_API_KEY !== undefined) { + server.configureArcApiKey(ARC_API_KEY) + } + + const chaintracksUrl = + CHAINTRACKS_URL ?? + (boolEnv('USE_ARCADE_CHAINTRACKS', ARCADE_URL !== undefined) ? ARCADE_URL : undefined) + if (chaintracksUrl !== undefined) { + if (typeof providerServer.configureChaintracks !== 'function') { + throw new TypeError( + 'CHAINTRACKS_URL/USE_ARCADE_CHAINTRACKS requires an @bsv/overlay-express version with configureChaintracks support' + ) } - - server.configureEnableBASMSync(boolEnv('BASM_ENABLED', false)) - const basmBlockPollIntervalMs = numberEnv('BASM_BLOCK_POLL_INTERVAL_MS') - if (basmBlockPollIntervalMs !== undefined) { - server.configureBASMBlockPollInterval(basmBlockPollIntervalMs) - } - const unprovenMaintenanceIntervalMs = numberEnv('UNPROVEN_MAINTENANCE_INTERVAL_MS') ?? 0 - const unprovenEvictionBlocks = numberEnv('UNPROVEN_EVICTION_BLOCKS') - if (unprovenMaintenanceIntervalMs > 0 || unprovenEvictionBlocks !== undefined) { - if (typeof providerServer.configureUnprovenMaintenance !== 'function') { - throw new Error('Unproven maintenance configuration requires an @bsv/overlay-express version with configureUnprovenMaintenance support') - } - providerServer.configureUnprovenMaintenance({ - intervalMs: unprovenMaintenanceIntervalMs, - thresholdBlocks: unprovenEvictionBlocks - }) + providerServer.configureChaintracks(chaintracksUrl, { + apiPrefix: CHAINTRACKS_API_PREFIX, + reorgStream: boolEnv('BASM_REORG_STREAM_ENABLED', true), + scanDepth: numberEnv('BASM_REORG_SCAN_DEPTH') + }) + } + + server.configureEnableBASMSync(boolEnv('BASM_ENABLED', false)) + const basmBlockPollIntervalMs = numberEnv('BASM_BLOCK_POLL_INTERVAL_MS') + if (basmBlockPollIntervalMs !== undefined) { + server.configureBASMBlockPollInterval(basmBlockPollIntervalMs) + } + const unprovenMaintenanceIntervalMs = numberEnv('UNPROVEN_MAINTENANCE_INTERVAL_MS') ?? 0 + const unprovenEvictionBlocks = numberEnv('UNPROVEN_EVICTION_BLOCKS') + if (unprovenMaintenanceIntervalMs > 0 || unprovenEvictionBlocks !== undefined) { + if (typeof providerServer.configureUnprovenMaintenance !== 'function') { + throw new TypeError( + 'Unproven maintenance configuration requires an @bsv/overlay-express version with configureUnprovenMaintenance support' + ) } - - server.configureHealth({ - contextProvider: () => ({ - providers: { - arc: ARC_API_KEY !== undefined, - arcade: ARCADE_URL !== undefined, - chaintracks: chaintracksUrl !== undefined, - }, - broadcast: { - throwOnBroadcastFailure: boolEnv('THROW_ON_BROADCAST_FAIL', true), - }, - basm: { - enabled: boolEnv('BASM_ENABLED', false), - reorgStreamEnabled: boolEnv('BASM_REORG_STREAM_ENABLED', true), - blockPollIntervalMs: basmBlockPollIntervalMs, - unprovenMaintenanceIntervalMs, - unprovenEvictionBlocks, - }, - }), + providerServer.configureUnprovenMaintenance({ + intervalMs: unprovenMaintenanceIntervalMs, + thresholdBlocks: unprovenEvictionBlocks }) - - // Decide what port you want the server to listen on. - server.configurePort(8080) - - // Connect to your SQL database with Knex - await server.configureKnex(KNEX_URL) - - // Also, be sure to connect to MongoDB - await server.configureMongo(MONGO_URL) - - // Here, you will configure the overlay topic managers and lookup services you want. - // - Topic managers decide what outputs can go in your overlay - // - Lookup services help people find things in your overlay - - // Protocols - server.configureTopicManager('tm_protomap', new ProtoMapTopicManager()) - server.configureLookupServiceWithMongo('ls_protomap', createProtoMapLookupService) - - // Certificates - server.configureTopicManager('tm_certmap', new CertMapTopicManager()) - server.configureLookupServiceWithMongo('ls_certmap', createCertMapLookupService) - - // Baskets - server.configureTopicManager('tm_basketmap', new BasketMapTopicManager()) - server.configureLookupServiceWithMongo('ls_basketmap', createBasketMapLookupService) - - // UHRP - server.configureTopicManager('tm_uhrp', new UHRPTopicManager()) - server.configureLookupServiceWithMongo('ls_uhrp', createUHRPLookupService) - - // Identity - server.configureTopicManager('tm_identity', new IdentityTopicManager()) - server.configureLookupServiceWithMongo('ls_identity', createIdentityLookupService) - - // MessageBox - server.configureTopicManager('tm_messagebox', new MessageBoxTopicManager()) - server.configureLookupServiceWithMongo('ls_messagebox', createMessageBoxLookupService) - - // UMP - server.configureTopicManager('tm_users', new UMPTopicManager()) - server.configureLookupServiceWithMongo('ls_users', createUMPLookupService) - - // HelloWorld - server.configureTopicManager('tm_helloworld', new HelloWorldTopicManager()) - server.configureLookupServiceWithMongo('ls_helloworld', createHelloWorldLookupService) - - // SlackThread - server.configureTopicManager('tm_slackthread', new SlackThreadsTopicManager()) - server.configureLookupServiceWithMongo('ls_slackthread', createSlackThreadsLookupService) - - // DesktopIntegrity - server.configureTopicManager('tm_desktopintegrity', new DesktopIntegrityTopicManager()) - server.configureLookupServiceWithMongo('ls_desktopintegrity', createDesktopIntegrityLookupService) - - // Fractionalize - server.configureTopicManager('tm_fractionalize', new FractionalizeTopicManager()) - server.configureLookupServiceWithMongo('ls_fractionalize', createFractionalizeLookupService) - - // SupplyChain - server.configureTopicManager('tm_supplychain', new SupplyChainTopicManager()) - server.configureLookupServiceWithMongo('ls_supplychain', createSupplyChainLookupService) - - // MonsterBattle - server.configureTopicManager('tm_monsterbattle', new MonsterBattleTopicManager()) - server.configureLookupServiceWithMongo('ls_monsterbattle', createMonsterBattleLookupService) - - // Any - server.configureTopicManager('tm_anytx', new AnyTopicManager()) - server.configureLookupServiceWithMongo('ls_anytx', createAnyLookupService) - - // Apps - server.configureTopicManager('tm_apps', new AppsTopicManager()) - server.configureLookupServiceWithMongo('ls_apps', createAppsLookupService) - - // DID - server.configureTopicManager('tm_did', new DIDTopicManager()) - server.configureLookupServiceWithMongo('ls_did', createDIDLookupService) - - // WalletConfig - server.configureTopicManager('tm_walletconfig', new WalletConfigTopicManager()) - server.configureLookupServiceWithMongo('ls_walletconfig', createWalletConfigLookupService) - - // TokenDemo - server.configureTopicManager('tm_tokendemo', new TokenDemoTopicManager()) - server.configureLookupServiceWithMongo('ls_tokendemo', createTokenDemoLookupService) - - // Mandala (BRC-92 regulated token) — verifier/admin wallet derived from the node identity key. - // NOTE: production must use an HSM/KMS-custodied verifier key (see spec follow-ups); this local - // wiring reuses SERVER_PRIVATE_KEY and an empty in-memory sanctions list. - const mandalaWallet = new ProtoWallet(PrivateKey.fromHex(SERVER_PRIVATE_KEY)) as unknown as WalletInterface - let mandalaStorage: MandalaStorageManager | undefined - const requireMandalaStorage = (): MandalaStorageManager => { - if (mandalaStorage === undefined) { - throw new Error('Mandala storage is not initialized') - } - return mandalaStorage + } + + server.configureHealth({ + contextProvider: () => ({ + providers: { + arc: ARC_API_KEY !== undefined, + arcade: ARCADE_URL !== undefined, + chaintracks: chaintracksUrl !== undefined + }, + broadcast: { + throwOnBroadcastFailure: boolEnv('THROW_ON_BROADCAST_FAIL', true) + }, + basm: { + enabled: boolEnv('BASM_ENABLED', false), + reorgStreamEnabled: boolEnv('BASM_REORG_STREAM_ENABLED', true), + blockPollIntervalMs: basmBlockPollIntervalMs, + unprovenMaintenanceIntervalMs, + unprovenEvictionBlocks + } + }) + }) + + // Decide what port you want the server to listen on. + server.configurePort(8080) + + // Connect to your SQL database with Knex + await server.configureKnex(KNEX_URL) + + // Also, be sure to connect to MongoDB + await server.configureMongo(MONGO_URL) + + // Here, you will configure the overlay topic managers and lookup services you want. + // - Topic managers decide what outputs can go in your overlay + // - Lookup services help people find things in your overlay + + // Protocols + server.configureTopicManager('tm_protomap', new ProtoMapTopicManager()) + server.configureLookupServiceWithMongo('ls_protomap', createProtoMapLookupService) + + // Certificates + server.configureTopicManager('tm_certmap', new CertMapTopicManager()) + server.configureLookupServiceWithMongo('ls_certmap', createCertMapLookupService) + + // Baskets + server.configureTopicManager('tm_basketmap', new BasketMapTopicManager()) + server.configureLookupServiceWithMongo('ls_basketmap', createBasketMapLookupService) + + // UHRP + server.configureTopicManager('tm_uhrp', new UHRPTopicManager()) + server.configureLookupServiceWithMongo('ls_uhrp', createUHRPLookupService) + + // Identity + server.configureTopicManager('tm_identity', new IdentityTopicManager()) + server.configureLookupServiceWithMongo('ls_identity', createIdentityLookupService) + + // MessageBox + server.configureTopicManager('tm_messagebox', new MessageBoxTopicManager()) + server.configureLookupServiceWithMongo('ls_messagebox', createMessageBoxLookupService) + + // UMP + server.configureTopicManager('tm_users', new UMPTopicManager()) + server.configureLookupServiceWithMongo('ls_users', createUMPLookupService) + + // HelloWorld + server.configureTopicManager('tm_helloworld', new HelloWorldTopicManager()) + server.configureLookupServiceWithMongo('ls_helloworld', createHelloWorldLookupService) + + // SlackThread + server.configureTopicManager('tm_slackthread', new SlackThreadsTopicManager()) + server.configureLookupServiceWithMongo('ls_slackthread', createSlackThreadsLookupService) + + // DesktopIntegrity + server.configureTopicManager('tm_desktopintegrity', new DesktopIntegrityTopicManager()) + server.configureLookupServiceWithMongo('ls_desktopintegrity', createDesktopIntegrityLookupService) + + // Fractionalize + server.configureTopicManager('tm_fractionalize', new FractionalizeTopicManager()) + server.configureLookupServiceWithMongo('ls_fractionalize', createFractionalizeLookupService) + + // SupplyChain + server.configureTopicManager('tm_supplychain', new SupplyChainTopicManager()) + server.configureLookupServiceWithMongo('ls_supplychain', createSupplyChainLookupService) + + // MonsterBattle + server.configureTopicManager('tm_monsterbattle', new MonsterBattleTopicManager()) + server.configureLookupServiceWithMongo('ls_monsterbattle', createMonsterBattleLookupService) + + // Any + server.configureTopicManager('tm_anytx', new AnyTopicManager()) + server.configureLookupServiceWithMongo('ls_anytx', createAnyLookupService) + + // Apps + server.configureTopicManager('tm_apps', new AppsTopicManager()) + server.configureLookupServiceWithMongo('ls_apps', createAppsLookupService) + + // DID + server.configureTopicManager('tm_did', new DIDTopicManager()) + server.configureLookupServiceWithMongo('ls_did', createDIDLookupService) + + // WalletConfig + server.configureTopicManager('tm_walletconfig', new WalletConfigTopicManager()) + server.configureLookupServiceWithMongo('ls_walletconfig', createWalletConfigLookupService) + + // TokenDemo + server.configureTopicManager('tm_tokendemo', new TokenDemoTopicManager()) + server.configureLookupServiceWithMongo('ls_tokendemo', createTokenDemoLookupService) + + // Mandala (BRC-92 regulated token) — verifier/admin wallet derived from the node identity key. + // NOTE: production must use an HSM/KMS-custodied verifier key (see spec follow-ups); this local + // wiring reuses SERVER_PRIVATE_KEY and an empty in-memory sanctions list. + const mandalaWallet = new ProtoWallet( + PrivateKey.fromHex(SERVER_PRIVATE_KEY) + ) as unknown as WalletInterface + let mandalaStorage: MandalaStorageManager | undefined + const requireMandalaStorage = (): MandalaStorageManager => { + if (mandalaStorage === undefined) { + throw new Error('Mandala storage is not initialized') } - server.configureTopicManager('tm_mandala', new MandalaTopicManager({ - verifierWallet: mandalaWallet, - screeningProvider: new InMemoryScreeningProvider([]), - adminWallet: mandalaWallet, - adminProtocolID: [2, 'mandala admin'] as [2, string], - stateStore: { - getAssetState: async (assetId) => await requireMandalaStorage().getAssetState(assetId), - getTokenRow: async (txid, outputIndex) => await requireMandalaStorage().getTokenRow(txid, outputIndex) - } - })) - server.configureLookupServiceWithMongo('ls_mandala', (db) => { - mandalaStorage = new MandalaStorageManager(db) - return createMandalaLookupService(mandalaWallet, mandalaStorage)(db) + return mandalaStorage + } + server.configureTopicManager( + 'tm_mandala', + new MandalaTopicManager({ + verifierWallet: mandalaWallet, + screeningProvider: new InMemoryScreeningProvider([]), + adminWallet: mandalaWallet, + adminProtocolID: [2, 'mandala admin'] as [2, string], + stateStore: { + getAssetState: async assetId => await requireMandalaStorage().getAssetState(assetId), + getTokenRow: async (txid, outputIndex) => + await requireMandalaStorage().getTokenRow(txid, outputIndex) + } }) + ) + server.configureLookupServiceWithMongo('ls_mandala', db => { + mandalaStorage = new MandalaStorageManager(db) + return createMandalaLookupService(mandalaWallet, mandalaStorage)(db) + }) - // For simple local deployments, sync can be disabled. - server.configureEnableGASPSync(process.env?.GASP_ENABLED === 'true') + // For simple local deployments, sync can be disabled. + server.configureEnableGASPSync(process.env?.GASP_ENABLED === 'true') - // Lastly, configure the engine and start the server! - await server.configureEngine() + // Lastly, configure the engine and start the server! + await server.configureEngine() - // Configure verbose request logging - server.configureVerboseRequestLogging(true) + // Configure verbose request logging + server.configureVerboseRequestLogging(true) - server.app.get('/version', (_req: unknown, res: { json: (body: unknown) => void }) => { - res.json(packageJson) - }) + server.app.get('/version', (_req: unknown, res: { json: (body: unknown) => void }) => { + res.json(packageJson) + }) - // Start the server - await server.start() + // Start the server + await server.start() } // Happy hacking :) // Wrap startup in a span so a slow/failed boot is visible in traces, and emit // structured ready/fatal events with timing. -tracer.startActiveSpan('overlay.bootstrap', async (span) => { - const startedAt = Date.now() - try { - await main() - const duration_ms = Date.now() - startedAt - span.setAttribute('node.name', process.env.NODE_NAME ?? 'unknown') - span.setStatus({ code: SpanStatusCode.OK }) - log.info({ operation: 'bootstrap', outcome: 'ok', duration_ms }, 'overlay-server started') - } catch (err) { - const duration_ms = Date.now() - startedAt - span.recordException(err as Error) - span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message }) - log.error({ operation: 'bootstrap', outcome: 'error', duration_ms, err }, 'overlay-server failed to start') - process.exitCode = 1 - } finally { - span.end() - } +tracer.startActiveSpan('overlay.bootstrap', async span => { + const startedAt = Date.now() + try { + await main() + const duration_ms = Date.now() - startedAt + span.setAttribute('node.name', process.env.NODE_NAME ?? 'unknown') + span.setStatus({ code: SpanStatusCode.OK }) + log.info({ operation: 'bootstrap', outcome: 'ok', duration_ms }, 'overlay-server started') + } catch (err) { + const duration_ms = Date.now() - startedAt + span.recordException(err as Error) + span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message }) + log.error( + { operation: 'bootstrap', outcome: 'error', duration_ms, err }, + 'overlay-server failed to start' + ) + process.exitCode = 1 + } finally { + span.end() + } }) diff --git a/infra/uhrp-server-basic/Dockerfile b/infra/uhrp-server-basic/Dockerfile index 58b8cfc8b..05ec26a34 100644 --- a/infra/uhrp-server-basic/Dockerfile +++ b/infra/uhrp-server-basic/Dockerfile @@ -32,6 +32,9 @@ USER node EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/ready`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + # Preload the OpenTelemetry bootstrap so auto-instrumentation patches modules # before app code is imported. CMD ["node", "--require", "./out/src/telemetry.js", "out/src/index.js"] diff --git a/infra/uhrp-server-basic/package-lock.json b/infra/uhrp-server-basic/package-lock.json index 09674e9e5..f6cb52828 100644 --- a/infra/uhrp-server-basic/package-lock.json +++ b/infra/uhrp-server-basic/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bsv/uhrp-lite", - "version": "0.1.8", + "version": "0.1.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bsv/uhrp-lite", - "version": "0.1.8", + "version": "0.1.9", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/auth-express-middleware": "^2.1.2", diff --git a/infra/uhrp-server-basic/package.json b/infra/uhrp-server-basic/package.json index 1fcd21f8c..c1e5f8af4 100644 --- a/infra/uhrp-server-basic/package.json +++ b/infra/uhrp-server-basic/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/uhrp-lite", - "version": "0.1.8", + "version": "0.1.9", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/infra/uhrp-server-basic/src/index.ts b/infra/uhrp-server-basic/src/index.ts index 15e640a3f..62c2e1600 100644 --- a/infra/uhrp-server-basic/src/index.ts +++ b/infra/uhrp-server-basic/src/index.ts @@ -25,38 +25,48 @@ import { readBodyLimitBytes, securityHeaders } from './security/edgePolicy' +import { createServiceHealth } from './serviceHealth' const SERVER_PRIVATE_KEY = process.env.SERVER_PRIVATE_KEY as string const HTTP_PORT = process.env.HTTP_PORT || 8080 +type RouteMethod = 'get' | 'put' | 'post' | 'patch' | 'delete' -const preAuthRateLimit = rateLimit(rateLimitOptions( - 'UHRP_PRE_AUTH_RATE_LIMIT', - { windowMs: 60_000, limit: 300 } -)) +const preAuthRateLimit = rateLimit( + rateLimitOptions('UHRP_PRE_AUTH_RATE_LIMIT', { windowMs: 60_000, limit: 300 }) +) -const authenticatedRateLimit = rateLimit(rateLimitOptions( - 'UHRP_AUTHENTICATED_RATE_LIMIT', - { windowMs: 60_000, limit: 1_000 }, - { keyGenerator: authenticatedIdentityKey } -)) +const authenticatedRateLimit = rateLimit( + rateLimitOptions( + 'UHRP_AUTHENTICATED_RATE_LIMIT', + { windowMs: 60_000, limit: 1_000 }, + { keyGenerator: authenticatedIdentityKey } + ) +) const app = express() +const serviceHealth = createServiceHealth() + app.disable('x-powered-by') configureTrustProxy(app) app.use(securityHeaders({ environmentPrefix: 'UHRP' })) -app.use(corsPolicy({ - environmentPrefix: 'UHRP', - methods: ['GET', 'PUT', 'POST', 'OPTIONS'] -})) +app.use( + corsPolicy({ + environmentPrefix: 'UHRP', + methods: ['GET', 'PUT', 'POST', 'OPTIONS'] + }) +) app.use(concurrencyLimit('UHRP', 100)) +serviceHealth.register(app) app.use(preAuthRateLimit) // Add CDN MIME type middleware before static middleware app.use(cdnMimeTypeMiddleware) app.use(express.static(path.join(__dirname, '../public'))) -app.use(bodyparser.json({ - limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), - type: 'application/json' -})) +app.use( + bodyparser.json({ + limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), + type: 'application/json' + }) +) app.use(bodyParserErrorHandler) app.use((req: Request, res: Response, next: NextFunction) => { @@ -70,121 +80,148 @@ app.use((req: Request, res: Response, next: NextFunction) => { }) // Unsecured pre-auth routes are added first -const preAuthRoutes = Object.values(routes.preAuth); -const postAuthRoutes = Object.values(routes.postAuth); +const preAuthRoutes = Object.values(routes.preAuth) +const postAuthRoutes = Object.values(routes.postAuth) // Cycle through pre-auth routes -preAuthRoutes.filter(route => (route as any).unsecured).forEach((route) => { - log.info({ operation: 'route.register', phase: 'pre_auth_unsecured', route_path: route.path, route_type: route.type }, 'adding route') - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( - route.path, - (route as any).middleware, - (route as any).func +preAuthRoutes + .filter(route => (route as any).unsecured) + .forEach(route => { + log.info( + { + operation: 'route.register', + phase: 'pre_auth_unsecured', + route_path: route.path, + route_type: route.type + }, + 'adding route' ) - } else { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) - } -}) + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) + } else { + app[route.type as RouteMethod](route.path, (route as any).func) + } + }) // Secured pre-auth routes are added next -preAuthRoutes.filter(route => !(route as any).unsecured).forEach((route) => { - log.info({ operation: 'route.register', phase: 'pre_auth_secured', route_path: route.path, route_type: route.type }, 'adding route') - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( - route.path, - (route as any).middleware, - (route as any).func +preAuthRoutes + .filter(route => !(route as any).unsecured) + .forEach(route => { + log.info( + { + operation: 'route.register', + phase: 'pre_auth_secured', + route_path: route.path, + route_type: route.type + }, + 'adding route' ) - } else { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) - } -}) - - // Auth is enforced from here forward - ; (async () => { - const wallet = await getWallet() - const authMiddleware = createAuthMiddleware({ - wallet, - allowUnauthenticated: false - }) - - const paymentMiddleware = createPaymentMiddleware({ - wallet, - calculateRequestPrice: async (req) => { - if (req.url === '/upload') { - const { fileSize, retentionPeriod } = (req.body as any) || {} - if (!fileSize || !retentionPeriod) return 0 - try { - const satoshis = await getPriceForFile({ fileSize: +fileSize, retentionPeriod: +retentionPeriod }) - return satoshis - } catch { - return 0 - } - } - if (req.url === '/renew') { - const { uhrpUrl, additionalMinutes } = (req.body as any) || {} - if (!uhrpUrl || !additionalMinutes) return 0 - try { - const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) - const satoshis = await getPriceForFile({ fileSize: +size, retentionPeriod: +additionalMinutes }) - return satoshis - } catch { - return 0 - } + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) + } else { + app[route.type as RouteMethod](route.path, (route as any).func) + } + }) + +// Auth is enforced from here forward +;(async () => { + const wallet = await getWallet() + const authMiddleware = createAuthMiddleware({ + wallet, + allowUnauthenticated: false + }) + + const paymentMiddleware = createPaymentMiddleware({ + wallet, + calculateRequestPrice: async req => { + if (req.url === '/upload') { + const { fileSize, retentionPeriod } = (req.body as any) || {} + if (!fileSize || !retentionPeriod) return 0 + try { + const satoshis = await getPriceForFile({ + fileSize: +fileSize, + retentionPeriod: +retentionPeriod + }) + return satoshis + } catch { + return 0 } - - return 0 } - }) - - app.use(authMiddleware); - app.use(authenticatedRateLimit) - app.use(paymentMiddleware) - - // Secured, post-auth routes are added - postAuthRoutes.forEach((route) => { - log.info({ operation: 'route.register', phase: 'post_auth', route_path: route.path, route_type: route.type }, 'adding route') - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( - route.path, - (route as any).middleware, - (route as any).func - ) - } else { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) + if (req.url === '/renew') { + const { uhrpUrl, additionalMinutes } = (req.body as any) || {} + if (!uhrpUrl || !additionalMinutes) return 0 + try { + const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) + const satoshis = await getPriceForFile({ + fileSize: +size, + retentionPeriod: +additionalMinutes + }) + return satoshis + } catch { + return 0 + } } - }) - app.use((req, res) => { - log.info({ operation: 'route.not_found', method: req.method, path: req.path }, 'Route not found') - res.status(404).json({ - status: 'error', - code: 'ERR_ROUTE_NOT_FOUND', - description: 'Route not found.' - }) - }) - - const server = app.listen(HTTP_PORT, () => { - const idKey = PrivateKey - .fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() - log.info( - { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: idKey }, - 'UHRP Storage Server listening' - ) - }) - configureHttpServer(server, 'UHRP', { - requestTimeoutMs: 5 * 60 * 1000, - headersTimeoutMs: 15_000, - keepAliveTimeoutMs: 5_000, - socketTimeoutMs: 5 * 60 * 1000, - maxRequestsPerSocket: 1_000 + return 0 + } + }) + + app.use(authMiddleware) + app.use(authenticatedRateLimit) + app.use(paymentMiddleware) + + // Secured, post-auth routes are added + postAuthRoutes.forEach(route => { + log.info( + { + operation: 'route.register', + phase: 'post_auth', + route_path: route.path, + route_type: route.type + }, + 'adding route' + ) + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) + } else { + app[route.type as RouteMethod](route.path, (route as any).func) + } + }) + + app.use((req, res) => { + log.info( + { operation: 'route.not_found', method: req.method, path: req.path }, + 'Route not found' + ) + res.status(404).json({ + status: 'error', + code: 'ERR_ROUTE_NOT_FOUND', + description: 'Route not found.' }) - - })().catch((error) => { - log.error({ operation: 'bootstrap', outcome: 'error', err: error }, 'UHRP Storage Server failed to start') - process.exit(1) - }); + }) + + serviceHealth.markReady() + const server = app.listen(HTTP_PORT, () => { + const idKey = PrivateKey.fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() + log.info( + { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: idKey }, + 'UHRP Storage Server listening' + ) + }) + configureHttpServer(server, 'UHRP', { + requestTimeoutMs: 5 * 60 * 1000, + headersTimeoutMs: 15_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 5 * 60 * 1000, + maxRequestsPerSocket: 1_000 + }) +})().catch(error => { + log.error( + { operation: 'bootstrap', outcome: 'error', err: error }, + 'UHRP Storage Server failed to start' + ) + process.exit(1) +}) diff --git a/infra/uhrp-server-basic/src/routes/list.ts b/infra/uhrp-server-basic/src/routes/list.ts index 4c8c21a17..f2993f21d 100644 --- a/infra/uhrp-server-basic/src/routes/list.ts +++ b/infra/uhrp-server-basic/src/routes/list.ts @@ -25,12 +25,12 @@ interface ListResponse { const listHandler = async (req: ListRequest, res: Response) => { if (!req.auth.identityKey || req.auth.identityKey === 'unknown') { - return res.status(400).json({ - status: 'error', - code: 'ERR_MISSING_IDENTITY_KEY', - description: 'Missing authfetch identityKey.' - }) - } + return res.status(400).json({ + status: 'error', + code: 'ERR_MISSING_IDENTITY_KEY', + description: 'Missing authfetch identityKey.' + }) + } try { const identityKey = req.auth.identityKey if (!identityKey) { @@ -66,7 +66,7 @@ const listHandler = async (req: ListRequest, res: Response) => { : '' const expiryTime = expiryTimeTag - ? parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) + ? Number.parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) : 0 if (Date.now() > expiryTime * 1000) { @@ -84,7 +84,10 @@ const listHandler = async (req: ListRequest, res: Response) => { uploads: result }) } catch (error) { - log.error({ operation: 'list.handle', outcome: 'error', err: error }, 'Error listing advertisements') + log.error( + { operation: 'list.handle', outcome: 'error', err: error }, + 'Error listing advertisements' + ) return res.status(500).json({ status: 'error', code: 'ERR_LIST', @@ -96,13 +99,13 @@ const listHandler = async (req: ListRequest, res: Response) => { export default { type: 'get', path: '/list', - summary: 'Lists all UHRP files (advertisements) matching the user\'s identityKey in transaction tags.', + summary: + "Lists all UHRP files (advertisements) matching the user's identityKey in transaction tags.", parameters: {}, exampleResponse: { status: 'success', uploads: [ { - uhrpUrl: 'uhrp://abcd1234...', expiryTime: 1691234567 } @@ -110,4 +113,4 @@ export default { }, errors: ['ERR_LIST'], func: listHandler -} \ No newline at end of file +} diff --git a/infra/uhrp-server-basic/src/routes/renew.ts b/infra/uhrp-server-basic/src/routes/renew.ts index 6e1565941..d00b0011d 100644 --- a/infra/uhrp-server-basic/src/routes/renew.ts +++ b/infra/uhrp-server-basic/src/routes/renew.ts @@ -28,17 +28,25 @@ interface RenewResponse { description?: string } -const renewHandler = async (req: RenewRequest, res: Response) => { - if (!req.auth.identityKey || req.auth.identityKey === 'unknown') { - return res.status(400).json({ - status: 'error', - code: 'ERR_MISSING_IDENTITY_KEY', - description: 'Missing authfetch identityKey.' - }) +const latestAdvertisement = (outputs: T[]): T | undefined => { + let latest: T | undefined + let latestExpiry = 0 + for (const output of outputs) { + const expiryTag = output.tags?.find(tag => tag.startsWith('expiry_time_')) + if (expiryTag == null) continue + const expiry = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + if (expiry > latestExpiry) { + latestExpiry = expiry + latest = output } + } + return latest +} + +const renewHandler = async (req: RenewRequest, res: Response) => { try { const { identityKey } = req.auth - if (!identityKey) { + if (!identityKey || identityKey === 'unknown') { return res.status(400).json({ status: 'error', code: 'ERR_MISSING_IDENTITY_KEY', @@ -68,9 +76,9 @@ const renewHandler = async (req: RenewRequest, res: Response) => } = await getMetadata(uhrpUrl, identityKey, limit, offset) // Convert to MS to create an ISO string - const newExpiryTimeSeconds = prevExpiryTime + (additionalMinutes * 60) + const newExpiryTimeSeconds = prevExpiryTime + additionalMinutes * 60 - const fileSizeNum = parseInt(size, 10) || 0 + const fileSizeNum = Number.parseInt(size, 10) || 0 let amount = 0 if (fileSizeNum > 0) { amount = await getPriceForFile({ @@ -81,9 +89,12 @@ const renewHandler = async (req: RenewRequest, res: Response) => // When multiple advertisements match, renew the one with the farthest expiry. const wallet = await getWallet() - const { outputs, BEEF, } = await wallet.listOutputs({ + const { outputs, BEEF } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`], + tags: [ + `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, + `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}` + ], tagQueryMode: 'all', includeTags: true, include: 'entire transactions', @@ -99,22 +110,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => }) } - // Finding the maxpiry file with the same url - let prevAdvertisement - // Farthest expiration time given in seconds - let maxpiry = 0 - for (const out of outputs) { - if (!out.tags) continue - const expiryTag = out.tags.find(t => t.startsWith('expiry_time_')) - if (!expiryTag) continue - - const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 - - if (expiryNum > maxpiry) { - maxpiry = expiryNum - prevAdvertisement = out - } - } + const prevAdvertisement = latestAdvertisement(outputs) if (!prevAdvertisement || !BEEF) { return res.status(404).json({ @@ -153,24 +149,30 @@ const renewHandler = async (req: RenewRequest, res: Response) => const uploaderTag = prevAdvertisement.tags.find(t => t.startsWith('uploader_identity_key_')) if (uploaderTag) newTags.push(uploaderTag) } - newTags.push(`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`) - newTags.push(`object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`) - newTags.push(`expiry_time_${newExpiryTimeSeconds}`) + newTags.push( + `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, + `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`, + `expiry_time_${newExpiryTimeSeconds}` + ) const { signableTransaction } = await wallet.createAction({ inputBEEF: BEEF, - inputs: [{ - outpoint: prevAdvertisement.outpoint, - unlockingScriptLength: 74, - inputDescription: 'Redeeming old advertisement' - }], - outputs: [{ - lockingScript: newLockingScript.toHex(), - satoshis: 1, - basket: 'uhrp advertisements', - outputDescription: 'UHRP advertisement token (renewed)', - tags: newTags - }], + inputs: [ + { + outpoint: prevAdvertisement.outpoint, + unlockingScriptLength: 74, + inputDescription: 'Redeeming old advertisement' + } + ], + outputs: [ + { + lockingScript: newLockingScript.toHex(), + satoshis: 1, + basket: 'uhrp advertisements', + outputDescription: 'UHRP advertisement token (renewed)', + tags: newTags + } + ], description: `Renew advertisement for uhrpUrl ${uhrpUrl}`, options: { randomizeOutputs: false @@ -190,8 +192,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => const unlockingScript = await unlocker.sign(partialTx, 0) const { tx, txid } = await wallet.signAction({ reference: signableTransaction.reference, - spends: - { + spends: { 0: { unlockingScript: unlockingScript.toHex() } @@ -229,7 +230,8 @@ const renewHandler = async (req: RenewRequest, res: Response) => export default { type: 'post', path: '/renew', - summary: 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', + summary: + 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', parameters: { uhrpUrl: 'The UHRP URL (e.g. "uhrp://somehash")', additionalMinutes: 'Number of minutes to extend' diff --git a/infra/uhrp-server-basic/src/serviceHealth.ts b/infra/uhrp-server-basic/src/serviceHealth.ts new file mode 100644 index 000000000..3543a6225 --- /dev/null +++ b/infra/uhrp-server-basic/src/serviceHealth.ts @@ -0,0 +1,24 @@ +import type { Express, Request, Response } from 'express' + +export interface ServiceHealth { + markReady: () => void + register: (app: Express) => void +} + +export const createServiceHealth = (): ServiceHealth => { + let ready = false + + return { + markReady: () => { + ready = true + }, + register: app => { + app.get('/health', (_req: Request, res: Response) => { + res.status(200).json({ status: 'ok', live: true }) + }) + app.get('/ready', (_req: Request, res: Response) => { + res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'starting', ready }) + }) + } + } +} diff --git a/infra/uhrp-server-basic/src/utils/getMetadata.ts b/infra/uhrp-server-basic/src/utils/getMetadata.ts index 15fa5bdee..55b60af46 100644 --- a/infra/uhrp-server-basic/src/utils/getMetadata.ts +++ b/infra/uhrp-server-basic/src/utils/getMetadata.ts @@ -1,13 +1,12 @@ import { getWallet } from './walletSingleton' import { Utils } from '@bsv/sdk' - interface FileMetadata { objectIdentifier: string name: string size: string contentType: string - expiryTime: number // minutes since the Unix epoch + expiryTime: number // minutes since the Unix epoch } /** @@ -18,11 +17,19 @@ interface FileMetadata { * @returns {Promise} An object containing file info. * @throws If no matching advertisement is found or GCS metadata fails. */ -export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, limit?: number, offset?: number): Promise { +export async function getMetadata( + uhrpUrl: string, + uploaderIdentityKey: string, + limit?: number, + offset?: number +): Promise { const wallet = await getWallet() const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `uploader_identity_key_${uploaderIdentityKey}`], + tags: [ + `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, + `uploader_identity_key_${uploaderIdentityKey}` + ], tagQueryMode: 'all', includeTags: true, limit: limit ?? 200, @@ -42,11 +49,13 @@ export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, const contentTypeTag = out.tags.find(t => t.startsWith('content_type_')) if (!objectIdTag || !expiryTag || !nameTag || !sizeTag || !contentTypeTag) continue - const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + const expiryNum = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 if (expiryNum > maxpiry) { maxpiry = expiryNum - objectIdentifier = Utils.toUTF8(Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex')) + objectIdentifier = Utils.toUTF8( + Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex') + ) name = nameTag size = sizeTag contentType = contentTypeTag @@ -54,7 +63,9 @@ export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, } if (!objectIdentifier || !name || !size || !contentType) { - throw new Error(`No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}`) + throw new Error( + `No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}` + ) } if (Date.now() > maxpiry * 1000) { diff --git a/infra/uhrp-server-basic/src/utils/getPriceForFile.ts b/infra/uhrp-server-basic/src/utils/getPriceForFile.ts index 9ce3421e7..416863490 100644 --- a/infra/uhrp-server-basic/src/utils/getPriceForFile.ts +++ b/infra/uhrp-server-basic/src/utils/getPriceForFile.ts @@ -14,14 +14,17 @@ interface PriceCalculationParams { * @param {PriceCalculationParams} params - Parameters for price calculation. * @returns {Promise} - The price in satoshis. */ -const getPriceForFile = async ({ retentionPeriod, fileSize }: PriceCalculationParams): Promise => { +const getPriceForFile = async ({ + retentionPeriod, + fileSize +}: PriceCalculationParams): Promise => { if (!PRICE_PER_GB_MO) { throw new Error('PRICE_PER_GB_MO is undefined') } - const pricePerGBMonth = parseFloat(PRICE_PER_GB_MO) - if (isNaN(pricePerGBMonth)) { - throw new Error('PRICE_PER_GB_MO must be a valid number') + const pricePerGBMonth = Number.parseFloat(PRICE_PER_GB_MO) + if (Number.isNaN(pricePerGBMonth)) { + throw new TypeError('PRICE_PER_GB_MO must be a valid number') } // File size is in bytes, convert to gigabytes @@ -36,24 +39,30 @@ const getPriceForFile = async ({ retentionPeriod, fileSize }: PriceCalculationPa // Get the exchange rate let exchangeRate: number try { - const { data } = await axios.get( - 'https://api.whatsonchain.com/v1/bsv/main/exchangerate' - ) - if (typeof data !== 'object' || isNaN(data.rate)) { - throw new Error('Invalid rate response') + const { data } = await axios.get('https://api.whatsonchain.com/v1/bsv/main/exchangerate') + if ( + data == null || + typeof data !== 'object' || + typeof data.rate !== 'number' || + !Number.isFinite(data.rate) + ) { + throw new TypeError('Invalid rate response') } exchangeRate = data.rate } catch (e) { exchangeRate = 30 - log.error({ operation: 'exchange_rate.fetch', outcome: 'error', fallback_rate: 30, err: e }, 'Exchange rate failed, using fallback rate') + log.error( + { operation: 'exchange_rate.fetch', outcome: 'error', fallback_rate: 30, err: e }, + 'Exchange rate failed, using fallback rate' + ) } // Exchange rate is in BSV, convert to satoshis const exchangeRateInSatoshis = 1 / (exchangeRate / 100000000) // Account for server overhead in our prices, so there is a minimum of 10 satoshis - let satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)); + const satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)) return satPrice } -export default getPriceForFile \ No newline at end of file +export default getPriceForFile diff --git a/infra/uhrp-server-basic/src/utils/getUploadURL.ts b/infra/uhrp-server-basic/src/utils/getUploadURL.ts index bf24654d1..a721b8da1 100644 --- a/infra/uhrp-server-basic/src/utils/getUploadURL.ts +++ b/infra/uhrp-server-basic/src/utils/getUploadURL.ts @@ -15,12 +15,12 @@ interface UploadResponse { requiredHeaders: Record } -export default async ({ +export default async function getUploadURL({ size, expiryTime, objectIdentifier, uploaderIdentityKey -}: UploadParams): Promise => { +}: UploadParams): Promise { const wallet = await getWallet() const customTime = new Date((expiryTime + 300) * 1000).toISOString() diff --git a/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts b/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts index 299a4260c..cedb8636f 100644 --- a/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts +++ b/infra/uhrp-server-basic/src/utils/mimeTypeMiddleware.ts @@ -12,6 +12,13 @@ import { CDN_ROOT } from './cdnObjectPath' const mimeTypeCache = new Map() const CACHE_TTL = 5 * 60 * 1000 // 5 minutes in milliseconds const cacheTimestamps = new Map() +const FILE_SIGNATURES = [ + { bytes: [0xff, 0xd8, 0xff], mimeType: 'image/jpeg' }, + { bytes: [0x89, 0x50, 0x4e, 0x47], mimeType: 'image/png' }, + { bytes: [0x47, 0x49, 0x46], mimeType: 'image/gif' }, + { bytes: [0x25, 0x50, 0x44, 0x46], mimeType: 'application/pdf' }, + { bytes: [0x50, 0x4b], mimeType: 'application/zip' } +] as const /** * Get MIME type from UHRP advertisement tags @@ -21,8 +28,8 @@ async function getMimeTypeFromAdvertisement(objectIdentifier: string): Promise t.startsWith('content_type_')) const expiryTag = output.tags.find(t => t.startsWith('expiry_time_')) - + if (contentTypeTag && expiryTag) { const expiryTime = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 - + // Only consider non-expired advertisements if (expiryTime > Date.now() / 1000 && expiryTime > maxExpiry) { maxExpiry = expiryTime @@ -65,7 +72,10 @@ async function getMimeTypeFromAdvertisement(objectIdentifier: string): Promise + signature.bytes.every((byte, index) => buffer[index] === byte) + )?.mimeType +} + +function isJson(text: string): boolean { + try { + JSON.parse(text) + return true + } catch { + return false + } +} + function detectMimeTypeFromContent(filePath: string): string { try { const buffer = fs.readFileSync(filePath, { encoding: null }) - const firstBytes = buffer.slice(0, 16) + const binaryMimeType = detectBinaryMimeType(buffer) + if (binaryMimeType != null) return binaryMimeType - // Check for common file signatures (magic bytes) - if (firstBytes[0] === 0xFF && firstBytes[1] === 0xD8 && firstBytes[2] === 0xFF) { - return 'image/jpeg' - } - if (firstBytes[0] === 0x89 && firstBytes[1] === 0x50 && firstBytes[2] === 0x4E && firstBytes[3] === 0x47) { - return 'image/png' - } - if (firstBytes[0] === 0x47 && firstBytes[1] === 0x49 && firstBytes[2] === 0x46) { - return 'image/gif' - } - if (firstBytes[0] === 0x25 && firstBytes[1] === 0x50 && firstBytes[2] === 0x44 && firstBytes[3] === 0x46) { - return 'application/pdf' - } - if (firstBytes[0] === 0x50 && firstBytes[1] === 0x4B) { - return 'application/zip' - } - - // Check if it's text-based content const textSample = buffer.slice(0, 512).toString('utf8', 0, Math.min(512, buffer.length)) - if (/^[\x20-\x7E\s]*$/.test(textSample)) { - if (textSample.trim().startsWith(' { if (error != null) { next() } }) } catch (error) { - log.error({ operation: 'mime.middleware', outcome: 'error', err: error }, 'Error in CDN MIME type middleware') + log.error( + { operation: 'mime.middleware', outcome: 'error', err: error }, + 'Error in CDN MIME type middleware' + ) next() } } diff --git a/infra/uhrp-server-basic/test/serviceHealth.test.js b/infra/uhrp-server-basic/test/serviceHealth.test.js new file mode 100644 index 000000000..c04eb17bc --- /dev/null +++ b/infra/uhrp-server-basic/test/serviceHealth.test.js @@ -0,0 +1,40 @@ +const express = require('express') +const { createServiceHealth } = require('../out/src/serviceHealth') + +const listen = async app => + await new Promise(resolve => { + const server = app.listen(0, '127.0.0.1', () => { + const address = server.address() + if (address === null || typeof address === 'string') throw new Error('Expected TCP address') + resolve({ origin: `http://127.0.0.1:${address.port}`, server }) + }) + }) + +const close = async server => + await new Promise((resolve, reject) => { + server.close(error => (error === undefined ? resolve() : reject(error))) + }) + +test('reports liveness immediately and readiness only after initialization', async () => { + const app = express() + const health = createServiceHealth() + health.register(app) + const { origin, server } = await listen(app) + + try { + const live = await fetch(`${origin}/health`) + expect(live.status).toBe(200) + await expect(live.json()).resolves.toEqual({ status: 'ok', live: true }) + + const starting = await fetch(`${origin}/ready`) + expect(starting.status).toBe(503) + await expect(starting.json()).resolves.toEqual({ status: 'starting', ready: false }) + + health.markReady() + const ready = await fetch(`${origin}/ready`) + expect(ready.status).toBe(200) + await expect(ready.json()).resolves.toEqual({ status: 'ready', ready: true }) + } finally { + await close(server) + } +}) diff --git a/infra/uhrp-server-cloud-bucket/Dockerfile b/infra/uhrp-server-cloud-bucket/Dockerfile index adb5524c9..297d690d8 100644 --- a/infra/uhrp-server-cloud-bucket/Dockerfile +++ b/infra/uhrp-server-cloud-bucket/Dockerfile @@ -38,6 +38,9 @@ USER node EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/ready`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + # Preload the OpenTelemetry bootstrap so auto-instrumentation patches modules # before app code is imported. CMD ["node", "--require", "./out/src/telemetry.js", "--max-http-header-size=512000", "out/src/index.js"] diff --git a/infra/uhrp-server-cloud-bucket/jest.config.js b/infra/uhrp-server-cloud-bucket/jest.config.js index c970cf9f8..f6e24ba72 100644 --- a/infra/uhrp-server-cloud-bucket/jest.config.js +++ b/infra/uhrp-server-cloud-bucket/jest.config.js @@ -4,23 +4,22 @@ module.exports = { roots: ['/src'], testMatch: ['**/__tests/**/*.test.js', '**/*.test.ts'], transform: { - '^.+\\.ts$': ['ts-jest', { - useESM: true, - tsconfig: { - module: 'ESNext', - moduleResolution: 'bundler' + '^.+\\.ts$': [ + 'ts-jest', + { + useESM: true, + tsconfig: { + module: 'ESNext', + moduleResolution: 'bundler' + } } - }], + ], '^.+\\.js$': 'babel-jest' }, moduleFileExtensions: ['ts', 'js', 'json'], extensionsToTreatAsEsm: ['.ts'], moduleNameMapper: { - '^(\.{1,2}/.*)\\.js$': '$1' + [String.raw`^(\.{1,2}/.*)\.js$`]: '$1' }, - collectCoverageFrom: [ - 'src/**/*.{ts,js}', - '!src/**/*.d.ts', - '!src/**/__tests__/**' - ] + collectCoverageFrom: ['src/**/*.{ts,js}', '!src/**/*.d.ts', '!src/**/__tests__/**'] } diff --git a/infra/uhrp-server-cloud-bucket/package-lock.json b/infra/uhrp-server-cloud-bucket/package-lock.json index 8c45eb269..ff21dd20d 100644 --- a/infra/uhrp-server-cloud-bucket/package-lock.json +++ b/infra/uhrp-server-cloud-bucket/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bsv/uhrp-storage-server", - "version": "0.2.10", + "version": "0.2.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bsv/uhrp-storage-server", - "version": "0.2.10", + "version": "0.2.11", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/auth-express-middleware": "^2.1.2", diff --git a/infra/uhrp-server-cloud-bucket/package.json b/infra/uhrp-server-cloud-bucket/package.json index e8c1b9fad..0f7d6dc65 100644 --- a/infra/uhrp-server-cloud-bucket/package.json +++ b/infra/uhrp-server-cloud-bucket/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/uhrp-storage-server", - "version": "0.2.10", + "version": "0.2.11", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts b/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts index 187d3164f..6716b92f1 100644 --- a/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts +++ b/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts @@ -13,140 +13,122 @@ * Writes: Environment Secrets named KEY (unprefixed), scoped to the selected environment. */ -import { spawnSync, execSync } from "node:child_process"; -import { existsSync, readFileSync, writeFileSync, unlinkSync } from "node:fs"; -import { join } from "node:path"; +import { spawnSync, execSync } from 'node:child_process' +import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs' +import { join } from 'node:path' -type EnvName = "staging" | "prod"; +type EnvName = 'staging' | 'prod' -const args = process.argv.slice(2); -const repo = getFlag("--repo") || getRepoFromGit(); -const envName = (getFlag("--env") as EnvName) || "staging"; -const createEnv = hasFlag("--create-env"); +const args = process.argv.slice(2) +const repo = getFlag('--repo') || getRepoFromGit() +const envName = (getFlag('--env') as EnvName) || 'staging' +const createEnv = hasFlag('--create-env') -if (!repo) die("Missing --repo owner/name"); -if (!["staging", "prod"].includes(envName)) - die("--env must be staging or prod"); +if (!repo) die('Missing --repo owner/name') +if (!['staging', 'prod'].includes(envName)) die('--env must be staging or prod') -ensureGhAuth(); +ensureGhAuth() -const secretsFile = join(process.cwd(), "secrets", `${envName}.env`); +const secretsFile = join(process.cwd(), 'secrets', `${envName}.env`) if (!existsSync(secretsFile)) { - die(`Secrets file not found: ${secretsFile}`); + die(`Secrets file not found: ${secretsFile}`) } -const kv = parseEnvFile(readFileSync(secretsFile, "utf8")); -const envLabel = envName === "prod" ? "production" : "staging"; // GitHub Environment name +const kv = parseEnvFile(readFileSync(secretsFile, 'utf8')) +const envLabel = envName === 'prod' ? 'production' : 'staging' // GitHub Environment name // Ensure the GitHub Environment exists -if (createEnv) ensureEnvironment(repo, envLabel); +if (createEnv) ensureEnvironment(repo, envLabel) -const keys = Object.keys(kv); -console.log( - `Syncing ${keys.length} secrets to ${repo} environment=${envLabel} (unprefixed names)` -); +const keys = Object.keys(kv) +console.log(`Syncing ${keys.length} secrets to ${repo} environment=${envLabel} (unprefixed names)`) -bulkSetSecrets(repo, envLabel, kv); +bulkSetSecrets(repo, envLabel, kv) -console.log(`Done. Pushed ${keys.length} secrets to ${repo} (${envLabel})`); +console.log(`Done. Pushed ${keys.length} secrets to ${repo} (${envLabel})`) function ensureGhAuth() { - const res = spawnSync("gh", ["auth", "status"], { stdio: "ignore" }); - if (res.status !== 0) die("GitHub CLI not authenticated. Run: gh auth login"); + const res = spawnSync('gh', ['auth', 'status'], { stdio: 'ignore' }) + if (res.status !== 0) die('GitHub CLI not authenticated. Run: gh auth login') } function ensureEnvironment(repository: string, env: string) { // 1) Check if the environment exists const check = spawnSync( - "gh", - ["api", `repos/${repository}/environments/${encodeURIComponent(env)}`], - { stdio: "ignore" } - ); - if (check.status === 0) return; // already exists + 'gh', + ['api', `repos/${repository}/environments/${encodeURIComponent(env)}`], + { stdio: 'ignore' } + ) + if (check.status === 0) return // already exists - console.log(`Creating environment '${env}' in ${repository}...`); + console.log(`Creating environment '${env}' in ${repository}...`) // 2) Create it (no body needed for basic create) const res = spawnSync( - "gh", - [ - "api", - "-X", - "PUT", - `repos/${repository}/environments/${encodeURIComponent(env)}`, - ], - { stdio: "inherit" } - ); - if (res.status !== 0) die(`Failed to create environment '${env}'.`); + 'gh', + ['api', '-X', 'PUT', `repos/${repository}/environments/${encodeURIComponent(env)}`], + { stdio: 'inherit' } + ) + if (res.status !== 0) die(`Failed to create environment '${env}'.`) } function parseEnvFile(src: string): Record { - const out: Record = {}; + const out: Record = {} for (const raw of src.split(/\r?\n/)) { - const line = raw.trim(); - if (!line || line.startsWith("#")) continue; - const idx = line.indexOf("="); - if (idx === -1) continue; - const key = line.slice(0, idx).trim(); - const val = line.slice(idx + 1).trim(); - if (!key) continue; - out[key] = val; + const line = raw.trim() + if (!line || line.startsWith('#')) continue + const idx = line.indexOf('=') + if (idx === -1) continue + const key = line.slice(0, idx).trim() + const val = line.slice(idx + 1).trim() + if (!key) continue + out[key] = val } - return out; + return out } function getFlag(name: string): string | undefined { - const i = args.indexOf(name); - if (i === -1 || i === args.length - 1) return undefined; - return args[i + 1]; + const i = args.indexOf(name) + if (i === -1 || i === args.length - 1) return undefined + return args[i + 1] } function hasFlag(name: string): boolean { - return args.includes(name); + return args.includes(name) } function die(msg: string): never { - console.error(`✖ ${msg}`); - process.exit(1); + console.error(`✖ ${msg}`) + process.exit(1) } function getRepoFromGit(): string { try { - const url = execSync("git config --get remote.origin.url") - .toString() - .trim(); - if (url.startsWith("git@github.com:")) { - return stripGitSuffix(url.slice("git@github.com:".length)); + const url = execSync('git config --get remote.origin.url').toString().trim() + if (url.startsWith('git@github.com:')) { + return stripGitSuffix(url.slice('git@github.com:'.length)) } - const parsed = new URL(url); - if (parsed.hostname === "github.com") { - const repo = parsed.pathname.split("/").filter(Boolean).join("/"); - return stripGitSuffix(repo); + const parsed = new URL(url) + if (parsed.hostname === 'github.com') { + const repo = parsed.pathname.split('/').filter(Boolean).join('/') + return stripGitSuffix(repo) } } catch {} - throw new Error( - "Unable to determine repo from git config. Pass --repo instead." - ); + throw new Error('Unable to determine repo from git config. Pass --repo instead.') } function stripGitSuffix(repo: string): string { - return repo.endsWith(".git") ? repo.slice(0, -4) : repo; + return repo.endsWith('.git') ? repo.slice(0, -4) : repo } -function bulkSetSecrets( - repository: string, - env: string, - kv: Record -) { - const tmp = join(process.cwd(), `.tmp_${env}_secrets_${Date.now()}.env`); - const lines = Object.entries(kv).map( - ([k, v]) => `${k}=${v.split("\n").join("\\n")}` - ); - writeFileSync(tmp, lines.join("\n")); - const res = spawnSync( - "gh", - ["secret", "set", "-R", repository, "-e", env, "-f", tmp], - { stdio: "inherit" } - ); +function bulkSetSecrets(repository: string, env: string, kv: Record) { + const tmp = join(process.cwd(), `.tmp_${env}_secrets_${Date.now()}.env`) + const lines = Object.entries(kv).map(([k, v]) => + `${k}=${v.replace(/\n/g, String.raw`\n`)}` + ) + writeFileSync(tmp, lines.join('\n')) + const res = spawnSync('gh', ['secret', 'set', '-R', repository, '-e', env, '-f', tmp], { + stdio: 'inherit' + }) try { - unlinkSync(tmp); + unlinkSync(tmp) } catch {} - if (res.status !== 0) die(`Bulk secret set failed for env ${env}`); + if (res.status !== 0) die(`Bulk secret set failed for env ${env}`) } diff --git a/infra/uhrp-server-cloud-bucket/src/index.ts b/infra/uhrp-server-cloud-bucket/src/index.ts index 69686e0cb..2080c547a 100644 --- a/infra/uhrp-server-cloud-bucket/src/index.ts +++ b/infra/uhrp-server-cloud-bucket/src/index.ts @@ -23,43 +23,56 @@ import { readBodyLimitBytes, securityHeaders } from './security/edgePolicy' +import { createServiceHealth } from './serviceHealth' const SERVER_PRIVATE_KEY = process.env.SERVER_PRIVATE_KEY as string const HTTP_PORT = process.env.HTTP_PORT || 8080 const NODE_ENV = process.env.NODE_ENV || 'development' +type RouteMethod = 'get' | 'put' | 'post' | 'patch' | 'delete' -const preAuthRateLimit = rateLimit(rateLimitOptions( - 'UHRP_PRE_AUTH_RATE_LIMIT', - { windowMs: 60_000, limit: 300 } -)) +const preAuthRateLimit = rateLimit( + rateLimitOptions('UHRP_PRE_AUTH_RATE_LIMIT', { windowMs: 60_000, limit: 300 }) +) -const authenticatedRateLimit = rateLimit(rateLimitOptions( - 'UHRP_AUTHENTICATED_RATE_LIMIT', - { windowMs: 60_000, limit: 1_000 }, - { keyGenerator: authenticatedIdentityKey } -)) +const authenticatedRateLimit = rateLimit( + rateLimitOptions( + 'UHRP_AUTHENTICATED_RATE_LIMIT', + { windowMs: 60_000, limit: 1_000 }, + { keyGenerator: authenticatedIdentityKey } + ) +) const app = express() +const serviceHealth = createServiceHealth() + app.disable('x-powered-by') configureTrustProxy(app) app.use(securityHeaders({ environmentPrefix: 'UHRP' })) -app.use(corsPolicy({ - environmentPrefix: 'UHRP', - methods: ['GET', 'POST', 'OPTIONS'] -})) +app.use( + corsPolicy({ + environmentPrefix: 'UHRP', + methods: ['GET', 'POST', 'OPTIONS'] + }) +) app.use(concurrencyLimit('UHRP', 200)) +serviceHealth.register(app) app.use(preAuthRateLimit) -app.use(bodyparser.json({ - limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), - type: 'application/json' -})) +app.use( + bodyparser.json({ + limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), + type: 'application/json' + }) +) app.use(bodyParserErrorHandler) app.use((req: Request, res: Response, next: NextFunction) => { log.info({ operation: 'request.in', method: req.method, url: req.url }, 'Incoming request') const originalJson = res.json.bind(res) res.json = (json: any) => { - log.info({ operation: 'response.json', method: req.method, url: req.url }, 'Outgoing JSON response') + log.info( + { operation: 'response.json', method: req.method, url: req.url }, + 'Outgoing JSON response' + ) return originalJson(json) } next() @@ -68,23 +81,24 @@ app.use((req: Request, res: Response, next: NextFunction) => { app.use(express.static('public')) // Unsecured pre-auth routes are added first -const preAuthRoutes = Object.values(routes.preAuth); -const postAuthRoutes = Object.values(routes.postAuth); +const preAuthRoutes = Object.values(routes.preAuth) +const postAuthRoutes = Object.values(routes.postAuth) // Cycle through pre-auth routes -preAuthRoutes.filter(route => (route as any).unsecured).forEach((route) => { - log.info({ operation: 'route.register', phase: 'pre_auth', secured: false, route_path: route.path }, 'Registering route') - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( - route.path, - (route as any).middleware, - (route as any).func +preAuthRoutes + .filter(route => (route as any).unsecured) + .forEach(route => { + log.info( + { operation: 'route.register', phase: 'pre_auth', secured: false, route_path: route.path }, + 'Registering route' ) - } else { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) - } -}) + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) + } else { + app[route.type as RouteMethod](route.path, (route as any).func) + } + }) // This ensures that HTTPS is used for uploads app.use((req: Request, res: Response, next: NextFunction) => { @@ -96,106 +110,113 @@ app.use((req: Request, res: Response, next: NextFunction) => { }) } next() -}); +}) // Secured pre-auth routes are added after the HTTPS redirect -preAuthRoutes.filter(route => !(route as any).unsecured).forEach((route) => { - log.info({ operation: 'route.register', phase: 'pre_auth', secured: true, route_path: route.path }, 'Registering route') - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( - route.path, - (route as any).middleware, - (route as any).func +preAuthRoutes + .filter(route => !(route as any).unsecured) + .forEach(route => { + log.info( + { operation: 'route.register', phase: 'pre_auth', secured: true, route_path: route.path }, + 'Registering route' ) - } else { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) - } -}) - - // Auth is enforced from here forward - ; (async () => { - const wallet = await getWallet() - const authMiddleware = createAuthMiddleware({ - wallet, - allowUnauthenticated: false - }) - - const paymentMiddleware = createPaymentMiddleware({ - wallet, - calculateRequestPrice: async (req) => { - - if (req.url === '/upload') { - const { fileSize, retentionPeriod } = (req.body as any) || {} - if (!fileSize || !retentionPeriod) return 0 - try { - const satoshis = await getPriceForFile({ fileSize: +fileSize, retentionPeriod: +retentionPeriod }) - return satoshis - } catch { - return 0 - } + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) + } else { + app[route.type as RouteMethod](route.path, (route as any).func) + } + }) + +// Auth is enforced from here forward +;(async () => { + const wallet = await getWallet() + const authMiddleware = createAuthMiddleware({ + wallet, + allowUnauthenticated: false + }) + + const paymentMiddleware = createPaymentMiddleware({ + wallet, + calculateRequestPrice: async req => { + if (req.url === '/upload') { + const { fileSize, retentionPeriod } = (req.body as any) || {} + if (!fileSize || !retentionPeriod) return 0 + try { + const satoshis = await getPriceForFile({ + fileSize: +fileSize, + retentionPeriod: +retentionPeriod + }) + return satoshis + } catch { + return 0 } - if (req.url === '/renew') { - const { uhrpUrl, additionalMinutes } = (req.body as any) || {} - if (!uhrpUrl || !additionalMinutes) return 0 - try { - const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) - const satoshis = await getPriceForFile({ fileSize: +size, retentionPeriod: +additionalMinutes }) - return satoshis - } catch { - return 0 - } + } + if (req.url === '/renew') { + const { uhrpUrl, additionalMinutes } = (req.body as any) || {} + if (!uhrpUrl || !additionalMinutes) return 0 + try { + const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) + const satoshis = await getPriceForFile({ + fileSize: +size, + retentionPeriod: +additionalMinutes + }) + return satoshis + } catch { + return 0 } - - return 0 } - }) - app.use(authMiddleware); - app.use(authenticatedRateLimit) - app.use(paymentMiddleware) - - // Secured, post-auth routes are added - postAuthRoutes.forEach((route) => { - log.info({ operation: 'route.register', phase: 'post_auth', secured: true, route_path: route.path }, 'Registering route') - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( - route.path, - (route as any).middleware, - (route as any).func - ) - } else { - app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) - } - }) + return 0 + } + }) - app.use((req, res) => { - log.warn({ operation: 'route.not_found', method: req.method, url: req.url }, 'Route not found') - res.status(404).json({ - status: 'error', - code: 'ERR_ROUTE_NOT_FOUND', - description: 'Route not found.' - }) - }) + app.use(authMiddleware) + app.use(authenticatedRateLimit) + app.use(paymentMiddleware) - const server = app.listen(HTTP_PORT, () => { - const identityKey = PrivateKey - .fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() - log.info( - { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: identityKey }, - 'UHRP Storage Server listening' - ) - }) - configureHttpServer(server, 'UHRP', { - requestTimeoutMs: 60_000, - headersTimeoutMs: 15_000, - keepAliveTimeoutMs: 5_000, - socketTimeoutMs: 60_000, - maxRequestsPerSocket: 1_000 + // Secured, post-auth routes are added + postAuthRoutes.forEach(route => { + log.info( + { operation: 'route.register', phase: 'post_auth', secured: true, route_path: route.path }, + 'Registering route' + ) + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) + } else { + app[route.type as RouteMethod](route.path, (route as any).func) + } + }) + + app.use((req, res) => { + log.warn({ operation: 'route.not_found', method: req.method, url: req.url }, 'Route not found') + res.status(404).json({ + status: 'error', + code: 'ERR_ROUTE_NOT_FOUND', + description: 'Route not found.' }) - - })().catch((error) => { - log.error({ operation: 'bootstrap', outcome: 'error', err: error }, 'UHRP Storage Server failed to start') - process.exit(1) - }); + }) + + serviceHealth.markReady() + const server = app.listen(HTTP_PORT, () => { + const identityKey = PrivateKey.fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() + log.info( + { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: identityKey }, + 'UHRP Storage Server listening' + ) + }) + configureHttpServer(server, 'UHRP', { + requestTimeoutMs: 60_000, + headersTimeoutMs: 15_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 60_000, + maxRequestsPerSocket: 1_000 + }) +})().catch(error => { + log.error( + { operation: 'bootstrap', outcome: 'error', err: error }, + 'UHRP Storage Server failed to start' + ) + process.exit(1) +}) diff --git a/infra/uhrp-server-cloud-bucket/src/routes/list.ts b/infra/uhrp-server-cloud-bucket/src/routes/list.ts index c38bf6f1d..4766c2195 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/list.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/list.ts @@ -59,7 +59,7 @@ const listHandler = async (req: ListRequest, res: Response) => { : '' const expiryTime = expiryTimeTag - ? parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) + ? Number.parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) : 0 if (Date.now() > expiryTime * 1000) { @@ -89,13 +89,13 @@ const listHandler = async (req: ListRequest, res: Response) => { export default { type: 'get', path: '/list', - summary: 'Lists all UHRP files (advertisements) matching the user\'s identityKey in transaction tags.', + summary: + "Lists all UHRP files (advertisements) matching the user's identityKey in transaction tags.", parameters: {}, exampleResponse: { status: 'success', uploads: [ { - uhrpUrl: 'uhrp://abcd1234...', expiryTime: 1691234567 } diff --git a/infra/uhrp-server-cloud-bucket/src/routes/renew.ts b/infra/uhrp-server-cloud-bucket/src/routes/renew.ts index 928e4b40d..842edce09 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/renew.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/renew.ts @@ -31,6 +31,21 @@ interface RenewResponse { description?: string } +const latestAdvertisement = (outputs: T[]): T | undefined => { + let latest: T | undefined + let latestExpiry = 0 + for (const output of outputs) { + const expiryTag = output.tags?.find(tag => tag.startsWith('expiry_time_')) + if (expiryTag == null) continue + const expiry = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + if (expiry > latestExpiry) { + latestExpiry = expiry + latest = output + } + } + return latest +} + const renewHandler = async (req: RenewRequest, res: Response) => { try { const { identityKey } = req.auth @@ -64,10 +79,10 @@ const renewHandler = async (req: RenewRequest, res: Response) => } = await getMetadata(uhrpUrl, identityKey, limit, offset) // Convert to MS to create an ISO string - const newExpiryTimeSeconds = prevExpiryTime + (additionalMinutes * 60) + const newExpiryTimeSeconds = prevExpiryTime + additionalMinutes * 60 const newCustomTimeIso = new Date(newExpiryTimeSeconds * 1000).toISOString() - const fileSizeNum = parseInt(size, 10) || 0 + const fileSizeNum = Number.parseInt(size, 10) || 0 let amount = 0 if (fileSizeNum > 0) { amount = await getPriceForFile({ @@ -78,9 +93,12 @@ const renewHandler = async (req: RenewRequest, res: Response) => // When multiple advertisements match, renew the one with the farthest expiry. const wallet = await getWallet() - const { outputs, BEEF, } = await wallet.listOutputs({ + const { outputs, BEEF } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`], + tags: [ + `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, + `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}` + ], tagQueryMode: 'all', includeTags: true, include: 'entire transactions', @@ -96,22 +114,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => }) } - // Finding the maxpiry file with the same url - let prevAdvertisement - // Farthest expiration time given in seconds - let maxpiry = 0 - for (const out of outputs) { - if (!out.tags) continue - const expiryTag = out.tags.find(t => t.startsWith('expiry_time_')) - if (!expiryTag) continue - - const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 - - if (expiryNum > maxpiry) { - maxpiry = expiryNum - prevAdvertisement = out - } - } + const prevAdvertisement = latestAdvertisement(outputs) if (!prevAdvertisement || !BEEF) { return res.status(404).json({ @@ -150,24 +153,30 @@ const renewHandler = async (req: RenewRequest, res: Response) => const uploaderTag = prevAdvertisement.tags.find(t => t.startsWith('uploader_identity_key_')) if (uploaderTag) newTags.push(uploaderTag) } - newTags.push(`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`) - newTags.push(`object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`) - newTags.push(`expiry_time_${newExpiryTimeSeconds}`) + newTags.push( + `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, + `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`, + `expiry_time_${newExpiryTimeSeconds}` + ) const { signableTransaction } = await wallet.createAction({ inputBEEF: BEEF, - inputs: [{ - outpoint: prevAdvertisement.outpoint, - unlockingScriptLength: 74, - inputDescription: 'Redeeming old advertisement' - }], - outputs: [{ - lockingScript: newLockingScript.toHex(), - satoshis: 1, - basket: 'uhrp advertisements', - outputDescription: 'UHRP advertisement token (renewed)', - tags: newTags - }], + inputs: [ + { + outpoint: prevAdvertisement.outpoint, + unlockingScriptLength: 74, + inputDescription: 'Redeeming old advertisement' + } + ], + outputs: [ + { + lockingScript: newLockingScript.toHex(), + satoshis: 1, + basket: 'uhrp advertisements', + outputDescription: 'UHRP advertisement token (renewed)', + tags: newTags + } + ], description: `Renew advertisement for uhrpUrl ${uhrpUrl}`, options: { randomizeOutputs: false @@ -187,8 +196,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => const unlockingScript = await unlocker.sign(partialTx, 0) const { tx, txid } = await wallet.signAction({ reference: signableTransaction.reference, - spends: - { + spends: { 0: { unlockingScript: unlockingScript.toHex() } @@ -208,7 +216,9 @@ const renewHandler = async (req: RenewRequest, res: Response) => await broadcaster.broadcast(Transaction.fromAtomicBEEF(tx)) // Setting the new expiry time in the actual database - await storage.bucket(GCP_BUCKET_NAME).file(`cdn/${objectIdentifier}`) + await storage + .bucket(GCP_BUCKET_NAME) + .file(`cdn/${objectIdentifier}`) .setMetadata({ customTime: newCustomTimeIso }) return res.status(200).json({ @@ -230,7 +240,8 @@ const renewHandler = async (req: RenewRequest, res: Response) => export default { type: 'post', path: '/renew', - summary: 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', + summary: + 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', parameters: { uhrpUrl: 'The UHRP URL (e.g. "uhrp://somehash")', additionalMinutes: 'Number of minutes to extend' diff --git a/infra/uhrp-server-cloud-bucket/src/serviceHealth.test.ts b/infra/uhrp-server-cloud-bucket/src/serviceHealth.test.ts new file mode 100644 index 000000000..20618deca --- /dev/null +++ b/infra/uhrp-server-cloud-bucket/src/serviceHealth.test.ts @@ -0,0 +1,42 @@ +import express from 'express' +import { expect, test } from '@jest/globals' +import type { Server } from 'node:http' +import { createServiceHealth } from './serviceHealth' + +const listen = async (app: express.Express): Promise<{ origin: string; server: Server }> => + await new Promise(resolve => { + const server = app.listen(0, '127.0.0.1', () => { + const address = server.address() + if (address === null || typeof address === 'string') throw new Error('Expected TCP address') + resolve({ origin: `http://127.0.0.1:${address.port}`, server }) + }) + }) + +const close = async (server: Server): Promise => + await new Promise((resolve, reject) => { + server.close(error => (error === undefined ? resolve() : reject(error))) + }) + +test('reports liveness immediately and readiness only after initialization', async () => { + const app = express() + const health = createServiceHealth() + health.register(app) + const { origin, server } = await listen(app) + + try { + const live = await fetch(`${origin}/health`) + expect(live.status).toBe(200) + await expect(live.json()).resolves.toEqual({ status: 'ok', live: true }) + + const starting = await fetch(`${origin}/ready`) + expect(starting.status).toBe(503) + await expect(starting.json()).resolves.toEqual({ status: 'starting', ready: false }) + + health.markReady() + const ready = await fetch(`${origin}/ready`) + expect(ready.status).toBe(200) + await expect(ready.json()).resolves.toEqual({ status: 'ready', ready: true }) + } finally { + await close(server) + } +}) diff --git a/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts b/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts new file mode 100644 index 000000000..3543a6225 --- /dev/null +++ b/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts @@ -0,0 +1,24 @@ +import type { Express, Request, Response } from 'express' + +export interface ServiceHealth { + markReady: () => void + register: (app: Express) => void +} + +export const createServiceHealth = (): ServiceHealth => { + let ready = false + + return { + markReady: () => { + ready = true + }, + register: app => { + app.get('/health', (_req: Request, res: Response) => { + res.status(200).json({ status: 'ok', live: true }) + }) + app.get('/ready', (_req: Request, res: Response) => { + res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'starting', ready }) + }) + } + } +} diff --git a/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts b/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts index 19788d43a..70df832c4 100644 --- a/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts +++ b/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts @@ -11,7 +11,7 @@ interface FileMetadata { name: string size: string contentType: string - expiryTime: number // minutes since the Unix epoch + expiryTime: number // minutes since the Unix epoch } /** @@ -22,11 +22,19 @@ interface FileMetadata { * @returns {Promise} An object containing file info. * @throws If no matching advertisement is found or GCS metadata fails. */ -export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, limit?: number, offset?: number): Promise { +export async function getMetadata( + uhrpUrl: string, + uploaderIdentityKey: string, + limit?: number, + offset?: number +): Promise { const wallet = await getWallet() const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `uploader_identity_key_${uploaderIdentityKey}`], + tags: [ + `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, + `uploader_identity_key_${uploaderIdentityKey}` + ], tagQueryMode: 'all', includeTags: true, limit: limit ?? 200, @@ -43,16 +51,20 @@ export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, const expiryTag = out.tags.find(t => t.startsWith('expiry_time_')) if (!objectIdTag || !expiryTag) continue - const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + const expiryNum = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 if (expiryNum > maxpiry) { maxpiry = expiryNum - objectIdentifier = Utils.toUTF8(Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex')) + objectIdentifier = Utils.toUTF8( + Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex') + ) } } if (!objectIdentifier) { - throw new Error(`No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}`) + throw new Error( + `No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}` + ) } if (Date.now() > maxpiry * 1000) { @@ -63,11 +75,7 @@ export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, const file = storage.bucket(GCP_BUCKET_NAME!).file(`cdn/${objectIdentifier}`) const [gcsMetadata] = await file.getMetadata() - const { - name = '', - size, - contentType = '', - } = gcsMetadata + const { name = '', size, contentType = '' } = gcsMetadata return { objectIdentifier, diff --git a/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts b/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts index 423adf68c..a2c66c9dd 100644 --- a/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts +++ b/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts @@ -14,14 +14,17 @@ interface PriceCalculationParams { * @param {PriceCalculationParams} params - Parameters for price calculation. * @returns {Promise} - The price in satoshis. */ -const getPriceForFile = async ({ retentionPeriod, fileSize }: PriceCalculationParams): Promise => { +const getPriceForFile = async ({ + retentionPeriod, + fileSize +}: PriceCalculationParams): Promise => { if (!PRICE_PER_GB_MO) { throw new Error('PRICE_PER_GB_MO is undefined') } - const pricePerGBMonth = parseFloat(PRICE_PER_GB_MO) - if (isNaN(pricePerGBMonth)) { - throw new Error('PRICE_PER_GB_MO must be a valid number') + const pricePerGBMonth = Number.parseFloat(PRICE_PER_GB_MO) + if (Number.isNaN(pricePerGBMonth)) { + throw new TypeError('PRICE_PER_GB_MO must be a valid number') } // File size is in bytes, convert to gigabytes @@ -36,24 +39,30 @@ const getPriceForFile = async ({ retentionPeriod, fileSize }: PriceCalculationPa // Get the exchange rate let exchangeRate: number try { - const { data } = await axios.get( - 'https://api.whatsonchain.com/v1/bsv/main/exchangerate' - ) - if (typeof data !== 'object' || isNaN(data.rate)) { - throw new Error('Invalid rate response') + const { data } = await axios.get('https://api.whatsonchain.com/v1/bsv/main/exchangerate') + if ( + data == null || + typeof data !== 'object' || + typeof data.rate !== 'number' || + !Number.isFinite(data.rate) + ) { + throw new TypeError('Invalid rate response') } exchangeRate = data.rate } catch (e) { exchangeRate = 30 - log.error({ operation: 'price.exchange_rate', outcome: 'error', fallback_rate: 30, err: e }, 'Exchange rate failed, using fallback rate') + log.error( + { operation: 'price.exchange_rate', outcome: 'error', fallback_rate: 30, err: e }, + 'Exchange rate failed, using fallback rate' + ) } // Exchange rate is in BSV, convert to satoshis const exchangeRateInSatoshis = 1 / (exchangeRate / 100000000) // Account for server overhead in our prices, so there is a minimum of 10 satoshis - let satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)); + const satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)) return satPrice } -export default getPriceForFile \ No newline at end of file +export default getPriceForFile diff --git a/infra/wab/Dockerfile b/infra/wab/Dockerfile index 28418855a..34eb936e6 100644 --- a/infra/wab/Dockerfile +++ b/infra/wab/Dockerfile @@ -49,6 +49,9 @@ EXPOSE 8080 USER node +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD ["node", "-e", "const port=process.env.PORT||'8080';fetch(`http://127.0.0.1:${port}/info`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + # Start command — preload the OpenTelemetry bootstrap so auto-instrumentation # patches modules before app code is imported. CMD ["node", "--require", "./dist/telemetry.js", "dist/server.js"] diff --git a/infra/wab/deploy/README.md b/infra/wab/deploy/README.md new file mode 100644 index 000000000..d0d6f4573 --- /dev/null +++ b/infra/wab/deploy/README.md @@ -0,0 +1,32 @@ +# WAB Kubernetes deployment + +These manifests are operator templates. They do not create credentials and they +do not deploy themselves. Keep the WAB image tag paired with the digest verified +by the repository's infrastructure release workflow. + +Create `wab-secrets` through the cluster's secret-management path with: + +- `db-password`; +- `mysql-root-password`; +- any enabled identity-provider credentials; +- any optional faucet/storage private material. + +Do not commit Secret manifests or literal values. Non-secret database coordinates +live in `wab-config`. Replace them when using a managed database and remove the +sample MySQL Deployment. + +The WAB process applies migrations before it listens. Kubernetes startup, +readiness, and liveness checks use the public `/info` endpoint; reaching it proves +the process completed startup and bound port 8080. The application container runs +as an unprivileged user with a read-only root filesystem and no Linux +capabilities. + +WAB remains a public protocol service. Wildcard, credential-free CORS is the +default so deployed apps, webviews, mobile clients, and future callers can use it. +Exact-origin allowlists and credentialed exact-origin deployments are explicit +operator choices; CSP is not an API authorization mechanism. + +Back up the database and record the current schema and image digest before a +rollout. Apply the database change first, wait for its probes, then update the WAB +digest. Roll back to the prior digest only when its schema compatibility is +verified; otherwise restore through the database recovery runbook. diff --git a/infra/wab/deploy/mysql-deployment.yaml b/infra/wab/deploy/mysql-deployment.yaml index 730ef247f..ee571fca6 100644 --- a/infra/wab/deploy/mysql-deployment.yaml +++ b/infra/wab/deploy/mysql-deployment.yaml @@ -21,30 +21,61 @@ spec: containers: - env: - name: MYSQL_DATABASE - value: exampledb + valueFrom: + configMapKeyRef: + name: wab-config + key: db-name - name: MYSQL_PASSWORD - value: examplepass + valueFrom: + secretKeyRef: + name: wab-secrets + key: db-password - name: MYSQL_ROOT_PASSWORD - value: example + valueFrom: + secretKeyRef: + name: wab-secrets + key: mysql-root-password - name: MYSQL_USER - value: exampleuser + valueFrom: + configMapKeyRef: + name: wab-config + key: db-user image: mysql:8.4.10@sha256:8dbcf531a03aade657e181b9cf2f1d1803ce621a1d55610cb44cb531ab7d7db6 name: mysql ports: - containerPort: 3306 name: mysql + startupProbe: + exec: + command: + - mysqladmin + - ping + - --host=127.0.0.1 + - --silent + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + tcpSocket: + port: mysql + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: mysql + periodSeconds: 30 resources: requests: cpu: 100m memory: 512Mi ephemeral-storage: 1Gi limits: + cpu: '1' memory: 1Gi ephemeral-storage: 2Gi volumeMounts: - mountPath: /var/lib/mysql name: db-data restartPolicy: Always + terminationGracePeriodSeconds: 60 volumes: - name: db-data persistentVolumeClaim: diff --git a/infra/wab/deploy/web-configmap.yaml b/infra/wab/deploy/web-configmap.yaml new file mode 100644 index 000000000..03eb546d4 --- /dev/null +++ b/infra/wab/deploy/web-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: wab-config +data: + db-client: mysql2 + db-user: wab + db-name: wab + db-host: mysql + db-port: '3306' diff --git a/infra/wab/deploy/web-deployment.yaml b/infra/wab/deploy/web-deployment.yaml index 8ed6c430f..6aa73c672 100644 --- a/infra/wab/deploy/web-deployment.yaml +++ b/infra/wab/deploy/web-deployment.yaml @@ -16,30 +16,88 @@ spec: service: web spec: automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 containers: - # General image from infra-release.yaml (GHCR). Pin to a released v* tag. + # General image from infra-release.yaml (GHCR). Keep tag and digest together. - image: ghcr.io/bsv-blockchain/wab:v1.4.10@sha256:b770900fcdfd151eca342d0e3d9b1ed49dc987f961b68d38e112f136d1fe10e5 + imagePullPolicy: IfNotPresent env: - - name: DB_CLIENT - value: mysql2 - - name: DB_USER - value: exampleuser - - name: DB_PASS - value: examplepass - - name: DB_NAME - value: exampledb - - name: DB_HOST - value: mysql + - name: DB_CLIENT + valueFrom: + configMapKeyRef: + name: wab-config + key: db-client + - name: DB_USER + valueFrom: + configMapKeyRef: + name: wab-config + key: db-user + - name: DB_PASS + valueFrom: + secretKeyRef: + name: wab-secrets + key: db-password + - name: DB_NAME + valueFrom: + configMapKeyRef: + name: wab-config + key: db-name + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: wab-config + key: db-host + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: wab-config + key: db-port + - name: NODE_ENV + value: production + - name: PORT + value: '8080' name: web ports: - containerPort: 8080 + name: http protocol: TCP + startupProbe: + httpGet: + path: /info + port: http + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /info + port: http + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /info + port: http + periodSeconds: 30 + timeoutSeconds: 5 resources: requests: cpu: 100m memory: 256Mi ephemeral-storage: 1Gi limits: + cpu: '1' memory: 512Mi ephemeral-storage: 2Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 restartPolicy: Always diff --git a/infra/wab/deploy/web-service.yaml b/infra/wab/deploy/web-service.yaml index e60b5efe0..d80143231 100644 --- a/infra/wab/deploy/web-service.yaml +++ b/infra/wab/deploy/web-service.yaml @@ -6,8 +6,8 @@ metadata: name: web spec: ports: - - name: "8080" + - name: http port: 8080 - targetPort: 8080 + targetPort: http selector: service: web diff --git a/infra/wab/jest.config.ts b/infra/wab/jest.config.ts index a2e1ddb0f..c20022252 100644 --- a/infra/wab/jest.config.ts +++ b/infra/wab/jest.config.ts @@ -1,34 +1,37 @@ -import type { JestConfigWithTsJest } from 'ts-jest'; +import type { JestConfigWithTsJest } from 'ts-jest' const config: JestConfigWithTsJest = { - verbose: true, - testEnvironment: 'node', - preset: 'ts-jest', - roots: ['/src'], - testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.ts$', - globalSetup: '/jest.globalSetup.ts', - setupFilesAfterEnv: ['/jest.setup.ts'], - transform: { - '^.+\\.ts$': ['ts-jest', { - tsconfig: { - module: 'commonjs', - moduleResolution: 'bundler', - types: ['node', 'jest'] - } - }] - }, - moduleFileExtensions: ['ts', 'js', 'json'], - modulePathIgnorePatterns: ['/out/'], - collectCoverage: false, - coverageDirectory: 'coverage', - collectCoverageFrom: [ - 'src/**/*.ts', - '!src/**/*.test.ts', - '!src/**/*.spec.ts', - '!src/__tests__/**', - '!src/server.ts' - ], - testTimeout: 30000 -}; + verbose: true, + testEnvironment: 'node', + preset: 'ts-jest', + roots: ['/src'], + testRegex: String.raw`(/__tests__/.*|(\.|/)(test|spec))\.ts$`, + globalSetup: '/jest.globalSetup.ts', + setupFilesAfterEnv: ['/jest.setup.ts'], + transform: { + '^.+\\.ts$': [ + 'ts-jest', + { + tsconfig: { + module: 'commonjs', + moduleResolution: 'bundler', + types: ['node', 'jest'] + } + } + ] + }, + moduleFileExtensions: ['ts', 'js', 'json'], + modulePathIgnorePatterns: ['/out/'], + collectCoverage: false, + coverageDirectory: 'coverage', + collectCoverageFrom: [ + 'src/**/*.ts', + '!src/**/*.test.ts', + '!src/**/*.spec.ts', + '!src/__tests__/**', + '!src/server.ts' + ], + testTimeout: 30000 +} -export default config; +export default config diff --git a/infra/wab/jest.globalSetup.ts b/infra/wab/jest.globalSetup.ts index 2fd3f00f9..5f31d864f 100644 --- a/infra/wab/jest.globalSetup.ts +++ b/infra/wab/jest.globalSetup.ts @@ -1,10 +1,10 @@ // Global setup - runs ONCE before all test files -export default async () => { - // Set environment variables for test environment - process.env.NODE_ENV = 'test'; - process.env.SERVER_PRIVATE_KEY = "0".repeat(64); - process.env.STORAGE_URL = "https://storage.example.com"; - process.env.BSV_NETWORK = "testnet"; - process.env.COMMISSION_FEE = "1000"; -}; +export default async function globalSetup() { + // Set environment variables for test environment + process.env.NODE_ENV = 'test' + process.env.SERVER_PRIVATE_KEY = '0'.repeat(64) + process.env.STORAGE_URL = 'https://storage.example.com' + process.env.BSV_NETWORK = 'testnet' + process.env.COMMISSION_FEE = '1000' +} diff --git a/infra/wab/package-lock.json b/infra/wab/package-lock.json index 8e6b5beca..8722fcb18 100644 --- a/infra/wab/package-lock.json +++ b/infra/wab/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bsv/wab-server", - "version": "1.4.11", + "version": "1.4.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bsv/wab-server", - "version": "1.4.11", + "version": "1.4.12", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/sdk": "^2.2.0", diff --git a/infra/wab/package.json b/infra/wab/package.json index 11255f834..5cb33aac8 100644 --- a/infra/wab/package.json +++ b/infra/wab/package.json @@ -1,7 +1,7 @@ { "name": "@bsv/wab-server", "private": true, - "version": "1.4.11", + "version": "1.4.12", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/infra/wab/src/auth-methods/DevConsoleAuthMethod.ts b/infra/wab/src/auth-methods/DevConsoleAuthMethod.ts index ec85709cf..88c05b2d4 100644 --- a/infra/wab/src/auth-methods/DevConsoleAuthMethod.ts +++ b/infra/wab/src/auth-methods/DevConsoleAuthMethod.ts @@ -1,6 +1,6 @@ -import { AuthMethod, AuthPayload, AuthResult } from "./AuthMethod"; -import { randomInt } from "node:crypto"; -import { log } from "../logger"; +import { AuthMethod, AuthPayload, AuthResult } from './AuthMethod' +import { randomInt } from 'node:crypto' +import { log } from '../logger' /** * DevConsoleAuthMethod @@ -9,140 +9,146 @@ import { log } from "../logger"; * This allows developers to authenticate without requiring external services like Twilio. */ export class DevConsoleAuthMethod extends AuthMethod { - public methodType = "DevConsole"; - - // In-memory storage for OTP codes (only for development) - private otpStorage: Map = new Map(); - - /** - * Generates a random 6-digit OTP and logs it to the console. - * Expects `payload.phoneNumber` (could be phone number, email, or any identifier). - * - * @param presentationKey - The user's prospective presentation key - * @param payload - Must include { phoneNumber } - * @returns AuthResult - */ - public async startAuth(presentationKey: string, payload: AuthPayload): Promise { - const phoneNumber = payload.phoneNumber; - if (!phoneNumber) { - return { success: false, message: "phoneNumber is required." }; - } - - // Generate a 6-digit OTP - const otp = randomInt(100000, 1000000).toString(); - - // Store OTP with 10-minute expiration - const expiresAt = Date.now() + (10 * 60 * 1000); - this.otpStorage.set(phoneNumber, { otp, expiresAt, presentationKey }); - - // Deliberately disclose only the development OTP. Never log the - // presentation key or the in-memory OTP store. - log.info( - { - operation: 'auth.dev_console.start', - identifier: phoneNumber, - expires_at: expiresAt, - otp - }, - 'Development OTP code generated' - ); - - return { - success: true, - message: `Development OTP sent for ${phoneNumber}. Check console logs.`, - data: { phoneNumber } - }; + public readonly methodType = 'DevConsole' + + // In-memory storage for OTP codes (only for development) + private readonly otpStorage: Map< + string, + { otp: string; expiresAt: number; presentationKey: string } + > = new Map() + + /** + * Generates a random 6-digit OTP and logs it to the console. + * Expects `payload.phoneNumber` (could be phone number, email, or any identifier). + * + * @param presentationKey - The user's prospective presentation key + * @param payload - Must include { phoneNumber } + * @returns AuthResult + */ + public async startAuth(presentationKey: string, payload: AuthPayload): Promise { + const phoneNumber = payload.phoneNumber + if (!phoneNumber) { + return { success: false, message: 'phoneNumber is required.' } } - /** - * Verifies the provided OTP against the stored value. - * Expects `payload.phoneNumber` and `payload.otp`. - * - * @param presentationKey - The user's prospective presentation key - * @param payload - Must include { phoneNumber, otp } - * @returns AuthResult - */ - public async completeAuth(presentationKey: string, payload: AuthPayload): Promise { - const phoneNumber = payload.phoneNumber; - const providedOtp = payload.otp; - - if (!phoneNumber || !providedOtp) { - return { - success: false, - message: "phoneNumber and otp are required." - }; - } - - log.debug( - { operation: 'auth.dev_console.complete', identifier: phoneNumber }, - 'Verifying development OTP' - ) - - const storedData = this.otpStorage.get(phoneNumber); - if (!storedData) { - return { - success: false, - message: "No OTP found for this session. Please start authentication first." - }; - } - - // Check if OTP has expired - if (Date.now() > storedData.expiresAt) { - this.otpStorage.delete(phoneNumber); - return { - success: false, - message: "OTP has expired. Please request a new one." - }; - } - - // Check if OTP matches - if (storedData.otp !== providedOtp) { - return { - success: false, - message: "Invalid OTP code." - }; - } - - // Clean up stored OTP after successful verification - this.otpStorage.delete(phoneNumber); - - log.info({ operation: 'auth.dev_console.complete', identifier: phoneNumber, outcome: 'success' }, 'Development auth successful'); - - return { - success: true, - message: `Development authentication successful for ${phoneNumber}.` - }; + // Generate a 6-digit OTP + const otp = randomInt(100000, 1000000).toString() + + // Store OTP with 10-minute expiration + const expiresAt = Date.now() + 10 * 60 * 1000 + this.otpStorage.set(phoneNumber, { otp, expiresAt, presentationKey }) + + // Deliberately disclose only the development OTP. Never log the + // presentation key or the in-memory OTP store. + log.info( + { + operation: 'auth.dev_console.start', + identifier: phoneNumber, + expires_at: expiresAt, + otp + }, + 'Development OTP code generated' + ) + + return { + success: true, + message: `Development OTP sent for ${phoneNumber}. Check console logs.`, + data: { phoneNumber } + } + } + + /** + * Verifies the provided OTP against the stored value. + * Expects `payload.phoneNumber` and `payload.otp`. + * + * @param presentationKey - The user's prospective presentation key + * @param payload - Must include { phoneNumber, otp } + * @returns AuthResult + */ + public async completeAuth(presentationKey: string, payload: AuthPayload): Promise { + const phoneNumber = payload.phoneNumber + const providedOtp = payload.otp + + if (!phoneNumber || !providedOtp) { + return { + success: false, + message: 'phoneNumber and otp are required.' + } + } + + log.debug( + { operation: 'auth.dev_console.complete', identifier: phoneNumber }, + 'Verifying development OTP' + ) + + const storedData = this.otpStorage.get(phoneNumber) + if (!storedData) { + return { + success: false, + message: 'No OTP found for this session. Please start authentication first.' + } } - /** - * Stores the identifier in the config object for future recognition. - * - * @param payload - Must include { phoneNumber } - * @returns string - */ - public buildConfigFromPayload(payload: AuthPayload): string { - return payload.phoneNumber; + // Check if OTP has expired + if (Date.now() > storedData.expiresAt) { + this.otpStorage.delete(phoneNumber) + return { + success: false, + message: 'OTP has expired. Please request a new one.' + } } - /** - * Checks if this identifier is already linked to the user. - * - * @param storedConfig - The stored configuration - * @param payload - Must include { phoneNumber } - */ - public isAlreadyLinked(storedConfig: Record, payload: AuthPayload): boolean { - return storedConfig.phoneNumber === payload.phoneNumber; + // Check if OTP matches + if (storedData.otp !== providedOtp) { + return { + success: false, + message: 'Invalid OTP code.' + } } - /** - * Utility method to clear expired OTPs (can be called periodically) - */ - public clearExpiredOtps(): void { - const now = Date.now(); - for (const [key, data] of this.otpStorage.entries()) { - if (now > data.expiresAt) { - this.otpStorage.delete(key); - } - } + // Clean up stored OTP after successful verification + this.otpStorage.delete(phoneNumber) + + log.info( + { operation: 'auth.dev_console.complete', identifier: phoneNumber, outcome: 'success' }, + 'Development auth successful' + ) + + return { + success: true, + message: `Development authentication successful for ${phoneNumber}.` + } + } + + /** + * Stores the identifier in the config object for future recognition. + * + * @param payload - Must include { phoneNumber } + * @returns string + */ + public buildConfigFromPayload(payload: AuthPayload): string { + return payload.phoneNumber + } + + /** + * Checks if this identifier is already linked to the user. + * + * @param storedConfig - The stored configuration + * @param payload - Must include { phoneNumber } + */ + public isAlreadyLinked(storedConfig: Record, payload: AuthPayload): boolean { + return storedConfig.phoneNumber === payload.phoneNumber + } + + /** + * Utility method to clear expired OTPs (can be called periodically) + */ + public clearExpiredOtps(): void { + const now = Date.now() + for (const [key, data] of this.otpStorage.entries()) { + if (now > data.expiresAt) { + this.otpStorage.delete(key) + } } + } } diff --git a/infra/wab/src/auth-methods/PersonaAuthMethod.ts b/infra/wab/src/auth-methods/PersonaAuthMethod.ts index 67034226e..7f018ac6e 100644 --- a/infra/wab/src/auth-methods/PersonaAuthMethod.ts +++ b/infra/wab/src/auth-methods/PersonaAuthMethod.ts @@ -5,61 +5,61 @@ * In a real scenario, you'd integrate with the Persona or Jumio SDK via webhooks, tokens, etc. */ -import { AuthMethod, AuthPayload, AuthResult } from "./AuthMethod"; +import { AuthMethod, AuthPayload, AuthResult } from './AuthMethod' export class PersonaAuthMethod extends AuthMethod { - public methodType = "PersonaID"; + public readonly methodType = 'PersonaID' - // Persona/Jumio config would go here - constructor(private personaConfig: { apiKey: string }) { - super(); - } + // Persona/Jumio config would go here + constructor(private readonly personaConfig: { apiKey: string }) { + super() + } - /** - * In a real scenario, we might start a Persona session for ID capture. - * We'll mock that we simply create a "verification session id". - */ - public async startAuth(_presentationKey: string, _payload: AuthPayload): Promise { - // Create mock "sessionId" - const sessionId = "persona-session-abc123"; + /** + * In a real scenario, we might start a Persona session for ID capture. + * We'll mock that we simply create a "verification session id". + */ + public async startAuth(_presentationKey: string, _payload: AuthPayload): Promise { + // Create mock "sessionId" + const sessionId = 'persona-session-abc123' - return { - success: true, - message: "Persona session created (mock).", - data: { sessionId } - }; + return { + success: true, + message: 'Persona session created (mock).', + data: { sessionId } } + } - /** - * Complete an ID verification check. We'll mock success if a certain mock ID is provided. - */ - public async completeAuth(presentationKey: string, payload: AuthPayload): Promise { - const sessionId = payload.sessionId; - // In real usage, we'd confirm the sessionId is verified with Persona's API. - if (sessionId === "persona-session-abc123") { - return { - success: true, - message: "ID verified successfully (mock)." - }; - } else { - return { - success: false, - message: "ID verification failed (mock)." - }; - } + /** + * Complete an ID verification check. We'll mock success if a certain mock ID is provided. + */ + public async completeAuth(presentationKey: string, payload: AuthPayload): Promise { + const sessionId = payload.sessionId + // In real usage, we'd confirm the sessionId is verified with Persona's API. + if (sessionId === 'persona-session-abc123') { + return { + success: true, + message: 'ID verified successfully (mock).' + } + } else { + return { + success: false, + message: 'ID verification failed (mock).' + } } + } - /** - * If ID verification is successful, store relevant data. - */ - public buildConfigFromPayload(payload: AuthPayload): string { - return payload.sessionId - } + /** + * If ID verification is successful, store relevant data. + */ + public buildConfigFromPayload(payload: AuthPayload): string { + return payload.sessionId + } - /** - * For demonstration, if stored config has the same sessionId, we consider it "already linked". - */ - public isAlreadyLinked(storedConfig: Record, payload: AuthPayload): boolean { - return storedConfig.personaSessionId === payload.sessionId; - } + /** + * For demonstration, if stored config has the same sessionId, we consider it "already linked". + */ + public isAlreadyLinked(storedConfig: Record, payload: AuthPayload): boolean { + return storedConfig.personaSessionId === payload.sessionId + } } diff --git a/infra/wab/src/auth-methods/TwilioAuthMethod.ts b/infra/wab/src/auth-methods/TwilioAuthMethod.ts index a611d01bb..de9be298f 100644 --- a/infra/wab/src/auth-methods/TwilioAuthMethod.ts +++ b/infra/wab/src/auth-methods/TwilioAuthMethod.ts @@ -1,25 +1,18 @@ -import { - AuthMethod, - AuthPayload, - AuthResult, - InvalidAuthPayloadError -} from "./AuthMethod"; -import twilio from "twilio"; -import { log } from "../logger"; +import { AuthMethod, AuthPayload, AuthResult, InvalidAuthPayloadError } from './AuthMethod' +import twilio from 'twilio' +import { log } from '../logger' -const E164_PHONE_NUMBER = /^\+[1-9]\d{7,14}$/; +const E164_PHONE_NUMBER = /^\+[1-9]\d{7,14}$/ function canonicalPhoneNumber(payload: AuthPayload): string { - if (typeof payload.phoneNumber !== "string") { - throw new InvalidAuthPayloadError("phoneNumber is required."); - } - const phoneNumber = payload.phoneNumber.trim(); - if (!E164_PHONE_NUMBER.test(phoneNumber)) { - throw new InvalidAuthPayloadError( - "phoneNumber must use canonical E.164 format." - ); - } - return phoneNumber; + if (typeof payload.phoneNumber !== 'string') { + throw new InvalidAuthPayloadError('phoneNumber is required.') + } + const phoneNumber = payload.phoneNumber.trim() + if (!E164_PHONE_NUMBER.test(phoneNumber)) { + throw new InvalidAuthPayloadError('phoneNumber must use canonical E.164 format.') + } + return phoneNumber } /** @@ -28,174 +21,181 @@ function canonicalPhoneNumber(payload: AuthPayload): string { * A concrete implementation of AuthMethod using Twilio Verify for phone verification. */ export class TwilioAuthMethod extends AuthMethod { - public methodType = "TwilioPhone"; - - private twilioClient: twilio.Twilio; - private verifyServiceSid: string; - - /** - * @param twilioConfig.accountSid - Your Twilio Account SID - * @param twilioConfig.authToken - Your Twilio Auth Token - * @param twilioConfig.verifyServiceSid - The Twilio Verify Service SID - */ - constructor( - private twilioConfig: { - accountSid: string; - authToken: string; - verifyServiceSid: string; - } - ) { - super(); - this.verifyServiceSid = twilioConfig.verifyServiceSid; - this.twilioClient = twilio(twilioConfig.accountSid, twilioConfig.authToken); + public readonly methodType = 'TwilioPhone' + + private readonly twilioClient: twilio.Twilio + private readonly verifyServiceSid: string + + /** + * @param twilioConfig.accountSid - Your Twilio Account SID + * @param twilioConfig.authToken - Your Twilio Auth Token + * @param twilioConfig.verifyServiceSid - The Twilio Verify Service SID + */ + constructor( + private readonly twilioConfig: { + accountSid: string + authToken: string + verifyServiceSid: string } - - /** - * Initiates the Twilio Verify flow by sending an SMS verification to the provided phone number. - * Expects `payload.phoneNumber`. - * - * @param presentationKey - The user's prospective presentation key (not necessarily in the DB yet). - * @param payload - Must include { phoneNumber } - * @returns AuthResult - */ - public async startAuth(_presentationKey: string, payload: AuthPayload): Promise { - let phoneNumber: string; - try { - phoneNumber = canonicalPhoneNumber(payload); - } catch (error) { - return { - success: false, - message: error instanceof Error ? error.message : "Invalid phoneNumber." - }; - } - - try { - const isVoipNumber = await this.isVoipNumber(phoneNumber); - if (isVoipNumber) { - return { - success: false, - message: "VOIP phone numbers are not supported for verification." - }; - } - } catch (error: any) { - log.error({ operation: 'auth.twilio.validate_phone', err: error, outcome: 'error' }, 'Error validating phone number'); - return { - success: false, - message: "Failed to validate phone number for verification." - }; - } - - try { - await this.twilioClient.verify.v2 - .services(this.verifyServiceSid) - .verifications.create({ - to: phoneNumber, - channel: "sms" - }); - - return { - success: true, - message: `Verification code sent to ${phoneNumber}.` - }; - } catch (error: any) { - log.error({ operation: 'auth.twilio.start', err: error, outcome: 'error' }, 'Error starting Twilio phone verification'); - return { - success: false, - message: "Failed to start Twilio phone verification." - }; - } + ) { + super() + this.verifyServiceSid = twilioConfig.verifyServiceSid + this.twilioClient = twilio(twilioConfig.accountSid, twilioConfig.authToken) + } + + /** + * Initiates the Twilio Verify flow by sending an SMS verification to the provided phone number. + * Expects `payload.phoneNumber`. + * + * @param presentationKey - The user's prospective presentation key (not necessarily in the DB yet). + * @param payload - Must include { phoneNumber } + * @returns AuthResult + */ + public async startAuth(_presentationKey: string, payload: AuthPayload): Promise { + let phoneNumber: string + try { + phoneNumber = canonicalPhoneNumber(payload) + } catch (error) { + return { + success: false, + message: error instanceof Error ? error.message : 'Invalid phoneNumber.' + } } - /** - * Completes the Twilio Verify flow by checking the provided code against Twilio's Verify service. - * Expects `payload.phoneNumber` and `payload.otp`. - * - * @param presentationKey - The user's prospective presentation key. - * @param payload - Must include { phoneNumber, otp } - * @returns AuthResult - */ - public async completeAuth(_presentationKey: string, payload: AuthPayload): Promise { - const providedOtp = payload.otp; - if (typeof providedOtp !== "string" || providedOtp.length === 0) { - return { - success: false, - message: "phoneNumber and otp are required." - }; - } - - let phoneNumber: string; - try { - phoneNumber = canonicalPhoneNumber(payload); - } catch (error) { - return { - success: false, - message: error instanceof Error ? error.message : "Invalid phoneNumber." - }; - } - - try { - // Attempt to verify the code - const verificationCheck = await this.twilioClient.verify.v2 - .services(this.verifyServiceSid) - .verificationChecks.create({ - to: phoneNumber, - code: providedOtp - }); - - if (verificationCheck.status === "approved") { - // Code is correct, phone verified - return { - success: true, - message: `Phone number ${phoneNumber} verified successfully.` - }; - } else { - // Code is incorrect or expired - return { - success: false, - message: `Verification code invalid or expired. (status=${verificationCheck.status})` - }; - } - } catch (error: any) { - log.error({ operation: 'auth.twilio.complete', err: error, outcome: 'error' }, 'Error completing Twilio phone verification'); - return { - success: false, - message: "Failed to complete Twilio phone verification." - }; + try { + const isVoipNumber = await this.isVoipNumber(phoneNumber) + if (isVoipNumber) { + return { + success: false, + message: 'VOIP phone numbers are not supported for verification.' } + } + } catch (error: any) { + log.error( + { operation: 'auth.twilio.validate_phone', err: error, outcome: 'error' }, + 'Error validating phone number' + ) + return { + success: false, + message: 'Failed to validate phone number for verification.' + } } - /** - * If verification is successful, store the phone number in the config object - * so that the user can be recognized by that number in the future. - * - * @param payload - Must include { phoneNumber } - * @returns Record - */ - public buildConfigFromPayload(payload: AuthPayload): string { - return canonicalPhoneNumber(payload); + try { + await this.twilioClient.verify.v2.services(this.verifyServiceSid).verifications.create({ + to: phoneNumber, + channel: 'sms' + }) + + return { + success: true, + message: `Verification code sent to ${phoneNumber}.` + } + } catch (error: any) { + log.error( + { operation: 'auth.twilio.start', err: error, outcome: 'error' }, + 'Error starting Twilio phone verification' + ) + return { + success: false, + message: 'Failed to start Twilio phone verification.' + } } - - /** - * Checks if this phone number is already linked to the user. - * - * @param storedConfig - * @param payload - */ - public isAlreadyLinked(storedConfig: Record, payload: AuthPayload): boolean { - return storedConfig.phoneNumber === payload.phoneNumber; + } + + /** + * Completes the Twilio Verify flow by checking the provided code against Twilio's Verify service. + * Expects `payload.phoneNumber` and `payload.otp`. + * + * @param presentationKey - The user's prospective presentation key. + * @param payload - Must include { phoneNumber, otp } + * @returns AuthResult + */ + public async completeAuth(_presentationKey: string, payload: AuthPayload): Promise { + const providedOtp = payload.otp + if (typeof providedOtp !== 'string' || providedOtp.length === 0) { + return { + success: false, + message: 'phoneNumber and otp are required.' + } } - private async isVoipNumber(phoneNumber: string): Promise { - const lookup = await this.twilioClient.lookups.v2 - .phoneNumbers(phoneNumber) - .fetch({ fields: "line_type_intelligence" }); - - const lineTypeIntelligence = lookup.lineTypeIntelligence as any; - const lineType = - typeof lineTypeIntelligence === "string" - ? lineTypeIntelligence - : lineTypeIntelligence?.lineType || lineTypeIntelligence?.line_type; + let phoneNumber: string + try { + phoneNumber = canonicalPhoneNumber(payload) + } catch (error) { + return { + success: false, + message: error instanceof Error ? error.message : 'Invalid phoneNumber.' + } + } - return (lineType || "").toLowerCase() === "voip"; + try { + // Attempt to verify the code + const verificationCheck = await this.twilioClient.verify.v2 + .services(this.verifyServiceSid) + .verificationChecks.create({ + to: phoneNumber, + code: providedOtp + }) + + if (verificationCheck.status === 'approved') { + // Code is correct, phone verified + return { + success: true, + message: `Phone number ${phoneNumber} verified successfully.` + } + } else { + // Code is incorrect or expired + return { + success: false, + message: `Verification code invalid or expired. (status=${verificationCheck.status})` + } + } + } catch (error: any) { + log.error( + { operation: 'auth.twilio.complete', err: error, outcome: 'error' }, + 'Error completing Twilio phone verification' + ) + return { + success: false, + message: 'Failed to complete Twilio phone verification.' + } } + } + + /** + * If verification is successful, store the phone number in the config object + * so that the user can be recognized by that number in the future. + * + * @param payload - Must include { phoneNumber } + * @returns Record + */ + public buildConfigFromPayload(payload: AuthPayload): string { + return canonicalPhoneNumber(payload) + } + + /** + * Checks if this phone number is already linked to the user. + * + * @param storedConfig + * @param payload + */ + public isAlreadyLinked(storedConfig: Record, payload: AuthPayload): boolean { + return storedConfig.phoneNumber === payload.phoneNumber + } + + private async isVoipNumber(phoneNumber: string): Promise { + const lookup = await this.twilioClient.lookups.v2 + .phoneNumbers(phoneNumber) + .fetch({ fields: 'line_type_intelligence' }) + + const lineTypeIntelligence = lookup.lineTypeIntelligence as any + const lineType = + typeof lineTypeIntelligence === 'string' + ? lineTypeIntelligence + : lineTypeIntelligence?.lineType || lineTypeIntelligence?.line_type + + return (lineType || '').toLowerCase() === 'voip' + } } diff --git a/infra/wallet-infra/Dockerfile b/infra/wallet-infra/Dockerfile index b408200a3..342c7bc5e 100644 --- a/infra/wallet-infra/Dockerfile +++ b/infra/wallet-infra/Dockerfile @@ -35,4 +35,7 @@ USER node EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 \ + CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "--import", "./out/src/telemetry.js", "out/src/index.js"] diff --git a/infra/wallet-infra/guides/kube_samples/README.md b/infra/wallet-infra/guides/kube_samples/README.md new file mode 100644 index 000000000..1322ee165 --- /dev/null +++ b/infra/wallet-infra/guides/kube_samples/README.md @@ -0,0 +1,25 @@ +# Wallet infrastructure Kubernetes samples + +These files demonstrate the workload contract; they are not a production +environment. Replace the checked-in application image with the release tag and +digest verified for the source revision being deployed. + +Create `wallet-secrets` and `mysql-secrets` through the operator's secret +manager. `wallet-secrets` supplies `SERVER_PRIVATE_KEY` and +`KNEX_DB_CONNECTION`; `mysql-secrets` supplies `MYSQL_ROOT_PASSWORD`. Never +commit either Secret object or a literal credential. + +The wallet application completes storage migration and initialization before it +starts serving port 8080. Its public root response is used for startup, +readiness, and liveness checks. The application runs without capabilities as a +non-root user on a read-only root filesystem. MySQL retains a writable PVC and +has separate startup, readiness, and liveness checks. + +Wallet Storage is a public protocol service. Its default browser policy remains +credential-free wildcard CORS, including opaque origins. Configure an exact +allowlist only when the deployment intentionally serves a closed caller set. + +Before rollout, record the current image digest and database schema, take and +verify a database backup, and confirm sufficient PVC capacity. Roll back the +application digest only when the prior version supports the current schema; +otherwise follow the database restore procedure first. diff --git a/infra/wallet-infra/guides/kube_samples/mysql.yaml b/infra/wallet-infra/guides/kube_samples/mysql.yaml index 84f896f09..067f07a1a 100644 --- a/infra/wallet-infra/guides/kube_samples/mysql.yaml +++ b/infra/wallet-infra/guides/kube_samples/mysql.yaml @@ -5,8 +5,8 @@ metadata: spec: ports: - - port: 3306 - targetPort: 3306 + - port: 3306 + targetPort: 3306 selector: app: mysql clusterIP: None @@ -28,42 +28,52 @@ spec: app: mysql spec: automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 containers: - - name: mysql - image: mysql:8.4.10@sha256:8dbcf531a03aade657e181b9cf2f1d1803ce621a1d55610cb44cb531ab7d7db6 - env: - - name: MYSQL_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: mysql-secrets - key: MYSQL_ROOT_PASSWORD - - name: MYSQL_DATABASE - value: wallet_storage - ports: - - containerPort: 3306 - name: mysql - volumeMounts: - - name: mysql-persistent-storage - mountPath: /var/lib/mysql - resources: - requests: - memory: "512Mi" - cpu: "250m" - ephemeral-storage: "1Gi" - limits: - memory: "1Gi" - cpu: "500m" - ephemeral-storage: "2Gi" - livenessProbe: - exec: - command: ["mysqladmin", "ping", "-h", "localhost"] - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 + - name: mysql + image: mysql:8.4.10@sha256:8dbcf531a03aade657e181b9cf2f1d1803ce621a1d55610cb44cb531ab7d7db6 + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secrets + key: MYSQL_ROOT_PASSWORD + - name: MYSQL_DATABASE + value: wallet_storage + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + resources: + requests: + memory: '512Mi' + cpu: '250m' + ephemeral-storage: '1Gi' + limits: + memory: '1Gi' + cpu: '500m' + ephemeral-storage: '2Gi' + livenessProbe: + exec: + command: ['mysqladmin', 'ping', '-h', 'localhost'] + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: mysql + periodSeconds: 10 + startupProbe: + exec: + command: ['mysqladmin', 'ping', '-h', 'localhost'] + failureThreshold: 30 + periodSeconds: 5 volumes: - - name: mysql-persistent-storage - persistentVolumeClaim: - claimName: mysql-pvc + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pvc --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/infra/wallet-infra/guides/kube_samples/wallet.yaml b/infra/wallet-infra/guides/kube_samples/wallet.yaml index 6cc256fa0..e6bbc9622 100644 --- a/infra/wallet-infra/guides/kube_samples/wallet.yaml +++ b/infra/wallet-infra/guides/kube_samples/wallet.yaml @@ -8,9 +8,9 @@ metadata: spec: type: NodePort ports: - - port: 80 - targetPort: 8080 - name: http + - port: 80 + targetPort: 8080 + name: http selector: app: wallet-infra --- @@ -32,25 +32,58 @@ spec: app: wallet-infra spec: automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 containers: - - name: wallet-infra - image: your-registry/wallet-infra:2.0.1 # Replace with your container registry path and version - ports: - - containerPort: 8080 - envFrom: - - configMapRef: - name: wallet-config - - secretRef: - name: wallet-secrets - resources: - requests: - cpu: 200m - memory: 256Mi - ephemeral-storage: "1Gi" - limits: - cpu: 500m - memory: 512Mi - ephemeral-storage: "2Gi" + - name: wallet-infra + image: ghcr.io/bsv-blockchain/wallet-infra:v2.0.13@sha256:373b7a1011ac5e822a8221cc01d50754458c0675413df2013206f6a5ea45a412 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http + envFrom: + - configMapRef: + name: wallet-config + - secretRef: + name: wallet-secrets + resources: + requests: + cpu: 200m + memory: 256Mi + ephemeral-storage: '1Gi' + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: '2Gi' + startupProbe: + httpGet: + path: / + port: http + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: / + port: http + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: / + port: http + periodSeconds: 30 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -58,22 +91,22 @@ metadata: name: wallet-ingress annotations: - alb.ingress.kubernetes.io/scheme: "internet-facing" - alb.ingress.kubernetes.io/target-type: "ip" + alb.ingress.kubernetes.io/scheme: 'internet-facing' + alb.ingress.kubernetes.io/target-type: 'ip' alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]' - alb.ingress.kubernetes.io/ssl-redirect: "443" + alb.ingress.kubernetes.io/ssl-redirect: '443' # Add your SSL certificate configuration here - external-dns.alpha.kubernetes.io/hostname: "store.example.com" + external-dns.alpha.kubernetes.io/hostname: 'store.example.com' spec: ingressClassName: alb rules: - - host: store.example.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: wallet-infra - port: - number: 80 + - host: store.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: wallet-infra + port: + number: 80 diff --git a/infra/wallet-infra/package-lock.json b/infra/wallet-infra/package-lock.json index 550927ce9..781c3e9c0 100644 --- a/infra/wallet-infra/package-lock.json +++ b/infra/wallet-infra/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bsv/wallet-infra", - "version": "2.0.13", + "version": "2.0.14", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bsv/wallet-infra", - "version": "2.0.13", + "version": "2.0.14", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@bsv/payment-express-middleware": "^2.1.1", diff --git a/infra/wallet-infra/package.json b/infra/wallet-infra/package.json index e34f37a7f..d58b60307 100644 --- a/infra/wallet-infra/package.json +++ b/infra/wallet-infra/package.json @@ -1,7 +1,7 @@ { "name": "@bsv/wallet-infra", "private": true, - "version": "2.0.13", + "version": "2.0.14", "engines": { "node": ">=24 <25", "npm": ">=11" diff --git a/package.json b/package.json index fa4611c2e..5338024a8 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "check-versions": "node scripts/check-versions.mjs", "typescript:check": "node scripts/typescript-toolchain.mjs", "health:baseline": "node scripts/repository-health.mjs --update-contract-baseline", - "health:check": "node --test scripts/*.test.mjs && pnpm docs:facts:check && node scripts/typescript-toolchain.mjs && node scripts/package-license-policy.mjs && node scripts/sync-service-rate-limit-policy.mjs --check && node scripts/sync-service-edge-policy.mjs --check && node scripts/repository-health.mjs", + "health:check": "node --test scripts/*.test.mjs && pnpm docs:facts:check && pnpm ops:check && node scripts/typescript-toolchain.mjs && node scripts/package-license-policy.mjs && node scripts/sync-service-rate-limit-policy.mjs --check && node scripts/sync-service-edge-policy.mjs --check && node scripts/repository-health.mjs", "health:report": "node scripts/repository-health.mjs --format markdown", "test:governance": "node scripts/test-governance.mjs", "build:property": "pnpm -r --filter '!@bsv/ts-stack' --filter '!docs-site' --filter '!example-paymail' run build", @@ -30,13 +30,15 @@ "codegen": "node scripts/generate-openapi-types.mjs", "codegen:check": "node scripts/generate-openapi-types.mjs --check", "conformance": "node conformance/runner/src/runner.js", - "docs:facts": "node scripts/generate-parity-matrix.mjs && node scripts/generate-stack-facts.mjs", + "docs:facts": "node scripts/generate-parity-matrix.mjs && node scripts/generate-stack-facts.mjs && pnpm ops:docs", "docs:facts:check": "node scripts/generate-parity-matrix.mjs --check && node scripts/generate-stack-facts.mjs --check && node scripts/documentation-policy.mjs", "docs:examples": "node scripts/check-documentation-examples.mjs", "docs:asyncapi": "node scripts/generate-asyncapi-html.mjs", "docs:dev": "pnpm --filter docs-site dev", "docs:build": "pnpm docs:facts:check && pnpm --filter docs-site build", - "docs:preview": "pnpm --filter docs-site preview" + "docs:preview": "pnpm --filter docs-site preview", + "ops:docs": "node scripts/service-operations.mjs", + "ops:check": "node scripts/service-operations.mjs --check" }, "devDependencies": { "@arethetypeswrong/core": "0.18.5", diff --git a/plans/GOVERNANCE.md b/plans/GOVERNANCE.md index 2e1054a56..b3e684810 100644 --- a/plans/GOVERNANCE.md +++ b/plans/GOVERNANCE.md @@ -1,5 +1,11 @@ # Governance +> **Historical planning record — superseded.** This document describes an +> unadopted cross-language governance model and references files and roles that +> are not current repository policy. Use [tracker #324](https://github.com/bsv-blockchain/ts-stack/issues/324), +> `docs/about/contributing.md`, the `governance/` registries, and enforced +> repository rules as the current sources of truth. + ## Mission Make the BSV Distributed Applications Stack — across TypeScript, Go, Python, and Rust — easier to maintain, cleaner to read, measurably faster, demonstrably more secure, and boringly reliable. TypeScript is the canonical reference. Specs are the contract. diff --git a/plans/HUMAN_QUESTION.md b/plans/HUMAN_QUESTION.md index 4103adc73..f6781a76b 100644 --- a/plans/HUMAN_QUESTION.md +++ b/plans/HUMAN_QUESTION.md @@ -1,5 +1,10 @@ # Questions for Human Review on ts-stack Documentation +> **Resolved archive.** The answers below informed the documentation repair +> program. They are preserved as decision history and are not an active task +> list. Current facts are generated or verified by repository CI and remaining +> work is tracked in [#324](https://github.com/bsv-blockchain/ts-stack/issues/324). + ## Package Organization and Counts - The top-level README.md claims 35 packages + conformance across 7 domains, with a detailed map. docs/packages/index.md claims 27 production-ready. Which count and structure should be canonical? Should we consolidate into one source of truth? Consolidate into one source of truth. 27 seems right, because we condensed overlay topics into one package. @@ -40,4 +45,4 @@ Yes that's a good reference URL. - Conformance vectors coverage claims - do they match current META.json and vectors/ ? We should check. -Please review and provide guidance on these. I will update docs to remove any unverified claims in the meantime. \ No newline at end of file +Please review and provide guidance on these. I will update docs to remove any unverified claims in the meantime. diff --git a/plans/QUICK_OVERLAYS.md b/plans/QUICK_OVERLAYS.md index 1cd458b09..bedd67da2 100644 --- a/plans/QUICK_OVERLAYS.md +++ b/plans/QUICK_OVERLAYS.md @@ -1,6 +1,11 @@ # QUICK_OVERLAYS — fast, non-blocking identity & overlay lookups -Status: proposal. Owner: tbd. Target packages: `@bsv/sdk`, `@bsv/wallet-toolbox`, `@bsv/wallet-toolbox-mobile`, `@bsv/overlays/topics`, downstream apps (bsv-browser). +> **Unapproved proposal.** This is not current behavior or an implementation +> instruction. Before use it requires a current issue, fresh measurements on +> supported runtimes, public API and cache-consistency review, privacy/security +> analysis, compatibility tests, and maintainer approval. + +Status: proposal. Owner: TBD. Target packages: `@bsv/sdk`, `@bsv/wallet-toolbox`, `@bsv/wallet-toolbox-mobile`, `@bsv/overlays/topics`, downstream apps (bsv-browser). ## TL;DR diff --git a/plans/README.md b/plans/README.md new file mode 100644 index 000000000..16fca5bb8 --- /dev/null +++ b/plans/README.md @@ -0,0 +1,17 @@ +# Planning Records + +Files in this directory preserve decisions, proposals, and implementation +history. They are not automatically current policy. + +Current repository policy and work are governed by: + +- [tracker #324](https://github.com/bsv-blockchain/ts-stack/issues/324); +- `docs/about/contributing.md` and `.github/SECURITY.md`; +- `governance/` machine-readable registries and repository CI; and +- package and infrastructure operator guides. + +Each planning record must say whether it is historical, resolved, or still a +proposal. A proposal requires a current issue, compatibility and security +review, tests, and maintainer approval before implementation. When a planning +record disagrees with current code, CI, governance registries, or repository +policy, those current sources take precedence. diff --git a/plans/RELIABILITY.md b/plans/RELIABILITY.md index ac397e91e..4a6e7a881 100644 --- a/plans/RELIABILITY.md +++ b/plans/RELIABILITY.md @@ -1,6 +1,14 @@ # Reliability Levels -Every package in ts-stack carries a Reliability Level (RL) in its `BASELINE.md`. RL is a statement of current state, not a target. +> **Historical planning model — not current package status.** The repository +> does not presently assign authoritative RL values to every package. Use +> generated health/facts reports, CI, package documentation, and +> [tracker #324](https://github.com/bsv-blockchain/ts-stack/issues/324) for +> current evidence and remaining work. RL labels below remain a useful proposed +> vocabulary only. + +The proposed model assigned every package a Reliability Level (RL) in a +`BASELINE.md`. An RL would describe current evidence, not a target. | Level | Name | Definition | |-------|------|------------| diff --git a/plans/SECURITY.md b/plans/SECURITY.md index c83c05fb0..916ad4ab9 100644 --- a/plans/SECURITY.md +++ b/plans/SECURITY.md @@ -1,5 +1,10 @@ # Security Policy +> **Historical duplicate — superseded.** The current disclosure and security +> policy is [`.github/SECURITY.md`](../.github/SECURITY.md), supported by +> `docs/infrastructure/service-edge-security.md`, governance registries, and +> enforced security workflows. Do not maintain this planning copy as policy. + ## Supported Versions | Package tier | Supported | diff --git a/plans/TERATESTNET_CHAIN_CONFIG.md b/plans/TERATESTNET_CHAIN_CONFIG.md index ec4bf5404..9c66266ed 100644 --- a/plans/TERATESTNET_CHAIN_CONFIG.md +++ b/plans/TERATESTNET_CHAIN_CONFIG.md @@ -1,5 +1,11 @@ # Teratestnet (TTN) Chain Config +> **Unapproved proposal.** This document is not an instruction to rename a +> public chain identifier. Implementation requires a current issue, inventory +> against the latest code and deployed consumers, backward-compatibility and +> migration design, conformance fixtures, release notes, and maintainer +> approval. + Goal: make the TeraTestNet network work across the stack. The canonical chain identifier is **`ttn`** (renamed from the current `'teratest'` — see rename section). TeraTestNet is the full display name. Most code paths fall back to testnet behavior. diff --git a/scripts/service-operations.mjs b/scripts/service-operations.mjs new file mode 100644 index 000000000..89ec9f155 --- /dev/null +++ b/scripts/service-operations.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node + +import { existsSync } from 'node:fs' +import { readFile, readdir, writeFile } from 'node:fs/promises' +import { join, relative, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import YAML from 'yaml' + +export const ROOT = fileURLToPath(new URL('..', import.meta.url)) +const OUTPUT_PATH = join(ROOT, 'docs/reference/service-operations.md') +const DIGEST_IMAGE = /@sha256:[0-9a-f]{64}$/ +const SECRET_NAME = + /(password|private.?key|secret|token|api.?key|credential|knex.?url|mongo.?url|db.?pass|connection)/i + +const readJson = async path => JSON.parse(await readFile(path, 'utf8')) + +const podSpec = document => { + if (document.kind === 'Deployment') return document.spec?.template?.spec + if (document.kind === 'CronJob') return document.spec?.jobTemplate?.spec?.template?.spec + return undefined +} + +const yamlDocuments = async path => + YAML.parseAllDocuments(await readFile(path, 'utf8')) + .map(document => document.toJSON()) + .filter(document => document != null) + +const workloadDocuments = async path => + (await yamlDocuments(path)).filter(document => podSpec(document) !== undefined) + +const manifestFiles = async path => { + const files = [] + for (const entry of await readdir(path, { withFileTypes: true })) { + const entryPath = join(path, entry.name) + if (entry.isDirectory()) files.push(...(await manifestFiles(entryPath))) + else if (entry.isFile() && /\.ya?ml$/.test(entry.name)) files.push(entryPath) + } + return files +} + +const validateContainer = (container, prefix, errors) => { + if (!DIGEST_IMAGE.test(container.image ?? '')) { + errors.push(`${prefix} image must use a tag or name plus sha256 digest`) + } + for (const environment of container.env ?? []) { + if ( + SECRET_NAME.test(environment.name ?? '') && + typeof environment.value === 'string' && + environment.value !== '' + ) { + errors.push(`${prefix} ${environment.name} must not contain a literal value`) + } + } +} + +export async function validateServiceOperations(root = ROOT) { + const errors = [] + const registry = await readJson(join(root, 'governance/service-operations.json')) + const containers = await readJson(join(root, 'governance/container-images.json')) + + if (registry.schemaVersion !== 1) errors.push('service-operations schemaVersion must be 1') + const governedNames = containers.components.map(component => component.name).sort() + const serviceNames = registry.services.map(service => service.name).sort() + if (JSON.stringify(governedNames) !== JSON.stringify(serviceNames)) { + errors.push('service-operations services must exactly match container-images components') + } + + for (const manifestRoot of registry.manifestRoots ?? []) { + const absoluteRoot = join(root, manifestRoot) + if (!existsSync(absoluteRoot)) { + errors.push(`service-operations references missing manifest root ${manifestRoot}`) + continue + } + for (const manifest of await manifestFiles(absoluteRoot)) { + for (const document of await yamlDocuments(manifest)) { + const manifestName = `${relative(root, manifest)} ${document.kind}/${document.metadata?.name}` + if (document.kind === 'Secret') { + errors.push(`${manifestName} must not be checked in`) + } + if (document.kind === 'ConfigMap') { + for (const [name, value] of Object.entries(document.data ?? {})) { + if (SECRET_NAME.test(name) && typeof value === 'string' && value !== '') { + errors.push(`${manifestName} ${name} must not contain secret material`) + } + } + } + const spec = podSpec(document) + if (spec === undefined) continue + const workload = manifestName + if (spec.automountServiceAccountToken !== false) { + errors.push(`${workload} must disable service-account token mounting`) + } + for (const container of [...(spec.initContainers ?? []), ...(spec.containers ?? [])]) { + validateContainer(container, `${workload} container ${container.name}`, errors) + } + } + } + } + + for (const service of registry.services) { + const prefix = `service ${service.name}` + for (const field of [ + 'path', + 'port', + 'livenessPath', + 'readinessPath', + 'state', + 'migration', + 'backup', + 'operatorGuide' + ]) { + if (typeof service[field] !== 'string' || service[field].trim() === '') { + errors.push(`${prefix} must define ${field}`) + } + } + for (const path of [service.path, service.operatorGuide]) { + if (!existsSync(join(root, path))) errors.push(`${prefix} references missing ${path}`) + } + const dockerfilePath = join(root, service.path, 'Dockerfile') + if (!existsSync(dockerfilePath)) { + errors.push(`${prefix} is missing Dockerfile`) + continue + } + const dockerfile = await readFile(dockerfilePath, 'utf8') + if (!/^USER (?!root\b)\S+/m.test(dockerfile)) { + errors.push(`${prefix} Dockerfile must end in a non-root runtime user`) + } + if (!/^HEALTHCHECK /m.test(dockerfile)) { + errors.push(`${prefix} Dockerfile must define HEALTHCHECK`) + } + if (!dockerfile.includes(service.readinessPath)) { + errors.push(`${prefix} Dockerfile health check must use ${service.readinessPath}`) + } + } + + for (const workload of registry.applicationWorkloads) { + const documents = await workloadDocuments(join(root, workload.manifest)) + const deployment = documents.find(document => document.kind === 'Deployment') + const spec = deployment === undefined ? undefined : podSpec(deployment) + const prefix = `${workload.manifest} container ${workload.container}` + const container = spec?.containers?.find(item => item.name === workload.container) + if (container === undefined) { + errors.push(`${prefix} is missing`) + continue + } + validateContainer(container, prefix, errors) + if (spec.automountServiceAccountToken !== false) { + errors.push(`${prefix} must disable service-account token mounting`) + } + if (spec.securityContext?.runAsNonRoot !== true) { + errors.push(`${prefix} pod must require a non-root user`) + } + if (spec.securityContext?.seccompProfile?.type !== 'RuntimeDefault') { + errors.push(`${prefix} pod must use RuntimeDefault seccomp`) + } + if (container.securityContext?.allowPrivilegeEscalation !== false) { + errors.push(`${prefix} must disable privilege escalation`) + } + if (container.securityContext?.readOnlyRootFilesystem !== true) { + errors.push(`${prefix} must use a read-only root filesystem`) + } + if (!container.securityContext?.capabilities?.drop?.includes('ALL')) { + errors.push(`${prefix} must drop all Linux capabilities`) + } + for (const field of ['startupProbe', 'readinessProbe', 'livenessProbe', 'resources']) { + if (container[field] === undefined) errors.push(`${prefix} must define ${field}`) + } + } + + return { errors, registry } +} + +const escapeCell = value => + String(value) + .replaceAll('|', String.raw`\|`) + .replaceAll('\n', ' ') +const operatorGuideLink = path => + path.startsWith('docs/') + ? `../${path.slice('docs/'.length)}` + : `https://github.com/bsv-blockchain/ts-stack/blob/main/${path}` + +export function renderServiceOperations(registry) { + const rows = registry.services + .map( + service => + `| \`${escapeCell(service.name)}\` | ${escapeCell(service.port)} | ` + + `\`${escapeCell(service.livenessPath)}\` | \`${escapeCell(service.readinessPath)}\` | ` + + `[guide](${operatorGuideLink(escapeCell(service.operatorGuide))}) |` + ) + .join('\n') + const recovery = registry.services + .map( + service => + `### ${service.name}\n\n` + + `- State: ${service.state}\n` + + `- Migration/startup: ${service.migration}\n` + + `- Backup/restore: ${service.backup}\n` + + `- Operator guide: [${service.operatorGuide}](${operatorGuideLink(service.operatorGuide)})\n` + ) + .join('\n') + + return `--- +id: service-operations +title: 'Service Operations Contract' +kind: reference +version: '1.0.0' +last_updated: '${registry.lastReviewed}' +last_verified: '${registry.lastReviewed}' +review_cadence_days: 30 +status: stable +tags: [reference, infrastructure, operations, health, recovery] +--- + +# Service Operations Contract + +This page is generated from \`governance/service-operations.json\`. CI verifies +that all seven released services have a non-root, digest-pinned container with a +real health check and that checked-in application workloads retain startup, +readiness, liveness, resources, seccomp, dropped capabilities, a read-only root +filesystem, and secret indirection. + +## Runtime endpoints + +| Service | Port contract | Liveness | Readiness | Operations | +|---|---|---|---|---| +${rows} + +Health endpoints are public and non-sensitive. They do not replace protocol +authentication or rate limits. Public services retain wildcard, +credential-free CORS by default; CSP remains a separate document/UI policy. + +## State, migration, and recovery + +${recovery} +## Change procedure + +1. Change a service, Dockerfile, manifest, or operator guide. +2. Update \`governance/service-operations.json\` when the operational contract changes. +3. Run \`pnpm ops:docs\`, then \`pnpm ops:check\`. +4. Run the affected service tests and the full repository health, container, + documentation, security, and merge gates. +5. Deploy only through a separately authorized release and record the exact image + digest, probe evidence, migration result, backup, and rollback outcome. +` +} + +async function run() { + const check = process.argv.includes('--check') + if (process.argv.slice(2).some(argument => argument !== '--check')) { + throw new Error('Usage: node scripts/service-operations.mjs [--check]') + } + const { errors, registry } = await validateServiceOperations() + if (errors.length > 0) throw new Error(errors.join('\n')) + const content = renderServiceOperations(registry) + if (check) { + const committed = await readFile(OUTPUT_PATH, 'utf8') + if (committed !== content) throw new Error('service operations documentation is stale') + console.log(`Verified ${relative(ROOT, OUTPUT_PATH)}`) + } else { + await writeFile(OUTPUT_PATH, content) + console.log(`Generated ${relative(ROOT, OUTPUT_PATH)}`) + } +} + +const isMain = process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href +if (isMain) await run() diff --git a/scripts/service-operations.test.mjs b/scripts/service-operations.test.mjs new file mode 100644 index 000000000..c1c166f54 --- /dev/null +++ b/scripts/service-operations.test.mjs @@ -0,0 +1,12 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { validateServiceOperations } from './service-operations.mjs' + +test('released service operations and checked-in application workloads satisfy policy', async () => { + const { errors, registry } = await validateServiceOperations() + assert.deepEqual(errors, []) + assert.equal(registry.services.length, 7) + assert.equal(registry.applicationWorkloads.length, 3) + assert.equal(registry.manifestRoots.length, 3) + assert.ok(registry.services.every(service => service.publicProtocol === true)) +}) From e9d0e0847bd923c9f968a474be141c7ef58030f9 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Mon, 27 Jul 2026 22:32:27 -0700 Subject: [PATCH 2/4] fix: keep repository health check dependency-free --- .github/workflows/ci.yml | 3 +++ scripts/service-operations.mjs | 7 ++++--- scripts/service-operations.test.mjs | 14 ++++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2bc831b1..bef06d3f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,9 @@ jobs: - name: Reject high and critical dependency advisories run: pnpm audit:security + - name: Enforce the service operations contract + run: pnpm ops:check + - name: Check cross-package version references # Non-blocking. Multi-package patch PRs (one bump touches several packages whose # downstream deps can't reference the new versions until publish) will always diff --git a/scripts/service-operations.mjs b/scripts/service-operations.mjs index 89ec9f155..dc48e7a56 100644 --- a/scripts/service-operations.mjs +++ b/scripts/service-operations.mjs @@ -4,7 +4,6 @@ import { existsSync } from 'node:fs' import { readFile, readdir, writeFile } from 'node:fs/promises' import { join, relative, resolve } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' -import YAML from 'yaml' export const ROOT = fileURLToPath(new URL('..', import.meta.url)) const OUTPUT_PATH = join(ROOT, 'docs/reference/service-operations.md') @@ -20,10 +19,12 @@ const podSpec = document => { return undefined } -const yamlDocuments = async path => - YAML.parseAllDocuments(await readFile(path, 'utf8')) +const yamlDocuments = async path => { + const { default: YAML } = await import('yaml') + return YAML.parseAllDocuments(await readFile(path, 'utf8')) .map(document => document.toJSON()) .filter(document => document != null) +} const workloadDocuments = async path => (await yamlDocuments(path)).filter(document => podSpec(document) !== undefined) diff --git a/scripts/service-operations.test.mjs b/scripts/service-operations.test.mjs index c1c166f54..63c26992c 100644 --- a/scripts/service-operations.test.mjs +++ b/scripts/service-operations.test.mjs @@ -1,12 +1,18 @@ import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' import test from 'node:test' -import { validateServiceOperations } from './service-operations.mjs' +import { renderServiceOperations, ROOT } from './service-operations.mjs' -test('released service operations and checked-in application workloads satisfy policy', async () => { - const { errors, registry } = await validateServiceOperations() - assert.deepEqual(errors, []) +test('service operations registry renders all seven public services', async () => { + const registry = JSON.parse( + await readFile(join(ROOT, 'governance/service-operations.json'), 'utf8') + ) + const rendered = renderServiceOperations(registry) assert.equal(registry.services.length, 7) assert.equal(registry.applicationWorkloads.length, 3) assert.equal(registry.manifestRoots.length, 3) assert.ok(registry.services.every(service => service.publicProtocol === true)) + assert.match(rendered, /Public services retain wildcard,\ncredential-free CORS by default/) + for (const service of registry.services) assert.match(rendered, new RegExp(service.name)) }) From 291d398d2a6a3abff3e93deda3493958b07d428d Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Mon, 27 Jul 2026 22:37:07 -0700 Subject: [PATCH 3/4] fix: close Wave 25 Sonar findings --- infra/chaintracks-server/Dockerfile | 2 +- infra/message-box-server/Dockerfile | 2 +- infra/uhrp-server-basic/Dockerfile | 2 +- infra/uhrp-server-cloud-bucket/Dockerfile | 2 +- .../scripts/sync-secrets.ts | 86 +++++-- infra/wab/Dockerfile | 2 +- infra/wallet-infra/Dockerfile | 2 +- scripts/service-operations.mjs | 231 ++++++++++-------- 8 files changed, 208 insertions(+), 121 deletions(-) diff --git a/infra/chaintracks-server/Dockerfile b/infra/chaintracks-server/Dockerfile index a458a64dd..029b5bcb6 100644 --- a/infra/chaintracks-server/Dockerfile +++ b/infra/chaintracks-server/Dockerfile @@ -59,7 +59,7 @@ USER node EXPOSE 3011 3012 HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ - CMD ["node", "-e", "const port=process.env.PORT||'3011';fetch(`http://127.0.0.1:${port}/getInfo`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.PORT||'3011';fetch('http://127.0.0.1:'+port+'/getInfo').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] # Run the application with the OpenTelemetry bootstrap preloaded so # auto-instrumentation patches modules before app code is imported. diff --git a/infra/message-box-server/Dockerfile b/infra/message-box-server/Dockerfile index 2a64b7525..a9e2b7e38 100644 --- a/infra/message-box-server/Dockerfile +++ b/infra/message-box-server/Dockerfile @@ -53,7 +53,7 @@ EXPOSE 8080 USER node HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ - CMD ["node", "-e", "const port=process.env.PORT||process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/ready`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.PORT||process.env.HTTP_PORT||'8080';fetch('http://127.0.0.1:'+port+'/ready').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] # Start the app with the OpenTelemetry bootstrap preloaded (ESM --import) so # auto-instrumentation patches modules before they are imported. diff --git a/infra/uhrp-server-basic/Dockerfile b/infra/uhrp-server-basic/Dockerfile index 05ec26a34..23ac04d2a 100644 --- a/infra/uhrp-server-basic/Dockerfile +++ b/infra/uhrp-server-basic/Dockerfile @@ -33,7 +33,7 @@ USER node EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ - CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/ready`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch('http://127.0.0.1:'+port+'/ready').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] # Preload the OpenTelemetry bootstrap so auto-instrumentation patches modules # before app code is imported. diff --git a/infra/uhrp-server-cloud-bucket/Dockerfile b/infra/uhrp-server-cloud-bucket/Dockerfile index 297d690d8..053e5fee1 100644 --- a/infra/uhrp-server-cloud-bucket/Dockerfile +++ b/infra/uhrp-server-cloud-bucket/Dockerfile @@ -39,7 +39,7 @@ USER node EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ - CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/ready`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch('http://127.0.0.1:'+port+'/ready').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] # Preload the OpenTelemetry bootstrap so auto-instrumentation patches modules # before app code is imported. diff --git a/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts b/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts index 6716b92f1..93706aa66 100644 --- a/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts +++ b/infra/uhrp-server-cloud-bucket/scripts/sync-secrets.ts @@ -3,7 +3,8 @@ * * Requirements: * - GitHub CLI installed: https://cli.github.com/ - * - Logged in: `gh auth login`s + * - Logged in: `gh auth login` + * - Optional absolute CLI overrides: `GH_CLI_PATH`, `GIT_CLI_PATH` * * Usage: * npm run secrets:staging @@ -13,9 +14,19 @@ * Writes: Environment Secrets named KEY (unprefixed), scoped to the selected environment. */ -import { spawnSync, execSync } from 'node:child_process' -import { existsSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs' -import { join } from 'node:path' +import { execFileSync, spawnSync } from 'node:child_process' +import { + accessSync, + constants, + existsSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { isAbsolute, join } from 'node:path' type EnvName = 'staging' | 'prod' @@ -48,14 +59,14 @@ bulkSetSecrets(repo, envLabel, kv) console.log(`Done. Pushed ${keys.length} secrets to ${repo} (${envLabel})`) function ensureGhAuth() { - const res = spawnSync('gh', ['auth', 'status'], { stdio: 'ignore' }) + const res = spawnSync(getGhExecutable(), ['auth', 'status'], { stdio: 'ignore' }) if (res.status !== 0) die('GitHub CLI not authenticated. Run: gh auth login') } function ensureEnvironment(repository: string, env: string) { // 1) Check if the environment exists const check = spawnSync( - 'gh', + getGhExecutable(), ['api', `repos/${repository}/environments/${encodeURIComponent(env)}`], { stdio: 'ignore' } ) @@ -64,7 +75,7 @@ function ensureEnvironment(repository: string, env: string) { console.log(`Creating environment '${env}' in ${repository}...`) // 2) Create it (no body needed for basic create) const res = spawnSync( - 'gh', + getGhExecutable(), ['api', '-X', 'PUT', `repos/${repository}/environments/${encodeURIComponent(env)}`], { stdio: 'inherit' } ) @@ -101,7 +112,15 @@ function die(msg: string): never { function getRepoFromGit(): string { try { - const url = execSync('git config --get remote.origin.url').toString().trim() + const url = execFileSync( + getTrustedExecutable('GIT_CLI_PATH', [ + '/usr/bin/git', + '/opt/homebrew/bin/git', + '/usr/local/bin/git' + ]), + ['config', '--get', 'remote.origin.url'], + { encoding: 'utf8' } + ).trim() if (url.startsWith('git@github.com:')) { return stripGitSuffix(url.slice('git@github.com:'.length)) } @@ -118,17 +137,48 @@ function stripGitSuffix(repo: string): string { return repo.endsWith('.git') ? repo.slice(0, -4) : repo } +function getGhExecutable(): string { + return getTrustedExecutable('GH_CLI_PATH', [ + '/usr/bin/gh', + '/opt/homebrew/bin/gh', + '/usr/local/bin/gh' + ]) +} + +function getTrustedExecutable(environmentName: string, defaults: string[]): string { + const configured = process.env[environmentName] + const candidates = configured === undefined ? defaults : [configured] + for (const candidate of candidates) { + if (!isAbsolute(candidate)) continue + try { + accessSync(candidate, constants.X_OK) + return realpathSync(candidate) + } catch {} + } + const instruction = + configured === undefined + ? `Install the CLI in one of: ${defaults.join(', ')}` + : `${environmentName} must name an existing executable by absolute path` + throw new Error(instruction) +} + function bulkSetSecrets(repository: string, env: string, kv: Record) { - const tmp = join(process.cwd(), `.tmp_${env}_secrets_${Date.now()}.env`) - const lines = Object.entries(kv).map(([k, v]) => - `${k}=${v.replace(/\n/g, String.raw`\n`)}` - ) - writeFileSync(tmp, lines.join('\n')) - const res = spawnSync('gh', ['secret', 'set', '-R', repository, '-e', env, '-f', tmp], { - stdio: 'inherit' + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'uhrp-secrets-')) + const temporaryFile = join(temporaryDirectory, `${env}.env`) + const escapedNewline = String.raw`\n` + const lines = Object.entries(kv).map(([key, value]) => { + return `${key}=${value.split('\n').join(escapedNewline)}` }) + let status: number | null = null try { - unlinkSync(tmp) - } catch {} - if (res.status !== 0) die(`Bulk secret set failed for env ${env}`) + writeFileSync(temporaryFile, lines.join('\n'), { encoding: 'utf8', mode: 0o600 }) + status = spawnSync( + getGhExecutable(), + ['secret', 'set', '-R', repository, '-e', env, '-f', temporaryFile], + { stdio: 'inherit' } + ).status + } finally { + rmSync(temporaryDirectory, { recursive: true, force: true }) + } + if (status !== 0) die(`Bulk secret set failed for env ${env}`) } diff --git a/infra/wab/Dockerfile b/infra/wab/Dockerfile index 34eb936e6..96617b58e 100644 --- a/infra/wab/Dockerfile +++ b/infra/wab/Dockerfile @@ -50,7 +50,7 @@ EXPOSE 8080 USER node HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ - CMD ["node", "-e", "const port=process.env.PORT||'8080';fetch(`http://127.0.0.1:${port}/info`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.PORT||'8080';fetch('http://127.0.0.1:'+port+'/info').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] # Start command — preload the OpenTelemetry bootstrap so auto-instrumentation # patches modules before app code is imported. diff --git a/infra/wallet-infra/Dockerfile b/infra/wallet-infra/Dockerfile index 342c7bc5e..fda72509f 100644 --- a/infra/wallet-infra/Dockerfile +++ b/infra/wallet-infra/Dockerfile @@ -36,6 +36,6 @@ USER node EXPOSE 8080 HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 \ - CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch(`http://127.0.0.1:${port}/`).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + CMD ["node", "-e", "const port=process.env.HTTP_PORT||'8080';fetch('http://127.0.0.1:'+port+'/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] CMD ["node", "--import", "./out/src/telemetry.js", "out/src/index.js"] diff --git a/scripts/service-operations.mjs b/scripts/service-operations.mjs index dc48e7a56..c0cbf2a46 100644 --- a/scripts/service-operations.mjs +++ b/scripts/service-operations.mjs @@ -54,118 +54,155 @@ const validateContainer = (container, prefix, errors) => { } } -export async function validateServiceOperations(root = ROOT) { - const errors = [] - const registry = await readJson(join(root, 'governance/service-operations.json')) - const containers = await readJson(join(root, 'governance/container-images.json')) - +const validateRegistryShape = (registry, containers, errors) => { if (registry.schemaVersion !== 1) errors.push('service-operations schemaVersion must be 1') const governedNames = containers.components.map(component => component.name).sort() const serviceNames = registry.services.map(service => service.name).sort() if (JSON.stringify(governedNames) !== JSON.stringify(serviceNames)) { errors.push('service-operations services must exactly match container-images components') } +} - for (const manifestRoot of registry.manifestRoots ?? []) { - const absoluteRoot = join(root, manifestRoot) - if (!existsSync(absoluteRoot)) { - errors.push(`service-operations references missing manifest root ${manifestRoot}`) - continue - } - for (const manifest of await manifestFiles(absoluteRoot)) { - for (const document of await yamlDocuments(manifest)) { - const manifestName = `${relative(root, manifest)} ${document.kind}/${document.metadata?.name}` - if (document.kind === 'Secret') { - errors.push(`${manifestName} must not be checked in`) - } - if (document.kind === 'ConfigMap') { - for (const [name, value] of Object.entries(document.data ?? {})) { - if (SECRET_NAME.test(name) && typeof value === 'string' && value !== '') { - errors.push(`${manifestName} ${name} must not contain secret material`) - } - } - } - const spec = podSpec(document) - if (spec === undefined) continue - const workload = manifestName - if (spec.automountServiceAccountToken !== false) { - errors.push(`${workload} must disable service-account token mounting`) - } - for (const container of [...(spec.initContainers ?? []), ...(spec.containers ?? [])]) { - validateContainer(container, `${workload} container ${container.name}`, errors) - } - } +const validateConfigMap = (document, manifestName, errors) => { + for (const [name, value] of Object.entries(document.data ?? {})) { + if (SECRET_NAME.test(name) && typeof value === 'string' && value !== '') { + errors.push(`${manifestName} ${name} must not contain secret material`) } } +} - for (const service of registry.services) { - const prefix = `service ${service.name}` - for (const field of [ - 'path', - 'port', - 'livenessPath', - 'readinessPath', - 'state', - 'migration', - 'backup', - 'operatorGuide' - ]) { - if (typeof service[field] !== 'string' || service[field].trim() === '') { - errors.push(`${prefix} must define ${field}`) - } - } - for (const path of [service.path, service.operatorGuide]) { - if (!existsSync(join(root, path))) errors.push(`${prefix} references missing ${path}`) - } - const dockerfilePath = join(root, service.path, 'Dockerfile') - if (!existsSync(dockerfilePath)) { - errors.push(`${prefix} is missing Dockerfile`) - continue - } - const dockerfile = await readFile(dockerfilePath, 'utf8') - if (!/^USER (?!root\b)\S+/m.test(dockerfile)) { - errors.push(`${prefix} Dockerfile must end in a non-root runtime user`) - } - if (!/^HEALTHCHECK /m.test(dockerfile)) { - errors.push(`${prefix} Dockerfile must define HEALTHCHECK`) +const validateManifestWorkload = (spec, workload, errors) => { + if (spec.automountServiceAccountToken !== false) { + errors.push(`${workload} must disable service-account token mounting`) + } + for (const container of [...(spec.initContainers ?? []), ...(spec.containers ?? [])]) { + validateContainer(container, `${workload} container ${container.name}`, errors) + } +} + +const validateManifestDocument = (root, manifest, document, errors) => { + const manifestName = `${relative(root, manifest)} ${document.kind}/${document.metadata?.name}` + if (document.kind === 'Secret') errors.push(`${manifestName} must not be checked in`) + if (document.kind === 'ConfigMap') validateConfigMap(document, manifestName, errors) + const spec = podSpec(document) + if (spec !== undefined) validateManifestWorkload(spec, manifestName, errors) +} + +const validateManifestRoot = async (root, manifestRoot, errors) => { + const absoluteRoot = join(root, manifestRoot) + if (!existsSync(absoluteRoot)) { + errors.push(`service-operations references missing manifest root ${manifestRoot}`) + return + } + for (const manifest of await manifestFiles(absoluteRoot)) { + for (const document of await yamlDocuments(manifest)) { + validateManifestDocument(root, manifest, document, errors) } - if (!dockerfile.includes(service.readinessPath)) { - errors.push(`${prefix} Dockerfile health check must use ${service.readinessPath}`) + } +} + +const REQUIRED_SERVICE_FIELDS = [ + 'path', + 'port', + 'livenessPath', + 'readinessPath', + 'state', + 'migration', + 'backup', + 'operatorGuide' +] + +const validateServiceFields = (service, prefix, errors) => { + for (const field of REQUIRED_SERVICE_FIELDS) { + if (typeof service[field] !== 'string' || service[field].trim() === '') { + errors.push(`${prefix} must define ${field}`) } } +} + +const validateServiceDockerfile = async (root, service, prefix, errors) => { + const dockerfilePath = join(root, service.path, 'Dockerfile') + if (!existsSync(dockerfilePath)) { + errors.push(`${prefix} is missing Dockerfile`) + return + } + const dockerfile = await readFile(dockerfilePath, 'utf8') + if (!/^USER (?!root\b)\S+/m.test(dockerfile)) { + errors.push(`${prefix} Dockerfile must end in a non-root runtime user`) + } + if (!/^HEALTHCHECK /m.test(dockerfile)) { + errors.push(`${prefix} Dockerfile must define HEALTHCHECK`) + } + if (!dockerfile.includes(service.readinessPath)) { + errors.push(`${prefix} Dockerfile health check must use ${service.readinessPath}`) + } +} + +const validateService = async (root, service, errors) => { + const prefix = `service ${service.name}` + validateServiceFields(service, prefix, errors) + for (const path of [service.path, service.operatorGuide]) { + if (!existsSync(join(root, path))) errors.push(`${prefix} references missing ${path}`) + } + await validateServiceDockerfile(root, service, prefix, errors) +} +const validatePodSecurity = (spec, prefix, errors) => { + if (spec.automountServiceAccountToken !== false) { + errors.push(`${prefix} must disable service-account token mounting`) + } + if (spec.securityContext?.runAsNonRoot !== true) { + errors.push(`${prefix} pod must require a non-root user`) + } + if (spec.securityContext?.seccompProfile?.type !== 'RuntimeDefault') { + errors.push(`${prefix} pod must use RuntimeDefault seccomp`) + } +} + +const validateContainerSecurity = (container, prefix, errors) => { + if (container.securityContext?.allowPrivilegeEscalation !== false) { + errors.push(`${prefix} must disable privilege escalation`) + } + if (container.securityContext?.readOnlyRootFilesystem !== true) { + errors.push(`${prefix} must use a read-only root filesystem`) + } + if (!container.securityContext?.capabilities?.drop?.includes('ALL')) { + errors.push(`${prefix} must drop all Linux capabilities`) + } + for (const field of ['startupProbe', 'readinessProbe', 'livenessProbe', 'resources']) { + if (container[field] === undefined) errors.push(`${prefix} must define ${field}`) + } +} + +const validateApplicationWorkload = async (root, workload, errors) => { + const documents = await workloadDocuments(join(root, workload.manifest)) + const deployment = documents.find(document => document.kind === 'Deployment') + const spec = deployment === undefined ? undefined : podSpec(deployment) + const prefix = `${workload.manifest} container ${workload.container}` + const container = spec?.containers?.find(item => item.name === workload.container) + if (container === undefined) { + errors.push(`${prefix} is missing`) + return + } + validateContainer(container, prefix, errors) + validatePodSecurity(spec, prefix, errors) + validateContainerSecurity(container, prefix, errors) +} + +export async function validateServiceOperations(root = ROOT) { + const errors = [] + const registry = await readJson(join(root, 'governance/service-operations.json')) + const containers = await readJson(join(root, 'governance/container-images.json')) + + validateRegistryShape(registry, containers, errors) + for (const manifestRoot of registry.manifestRoots ?? []) { + await validateManifestRoot(root, manifestRoot, errors) + } + for (const service of registry.services) { + await validateService(root, service, errors) + } for (const workload of registry.applicationWorkloads) { - const documents = await workloadDocuments(join(root, workload.manifest)) - const deployment = documents.find(document => document.kind === 'Deployment') - const spec = deployment === undefined ? undefined : podSpec(deployment) - const prefix = `${workload.manifest} container ${workload.container}` - const container = spec?.containers?.find(item => item.name === workload.container) - if (container === undefined) { - errors.push(`${prefix} is missing`) - continue - } - validateContainer(container, prefix, errors) - if (spec.automountServiceAccountToken !== false) { - errors.push(`${prefix} must disable service-account token mounting`) - } - if (spec.securityContext?.runAsNonRoot !== true) { - errors.push(`${prefix} pod must require a non-root user`) - } - if (spec.securityContext?.seccompProfile?.type !== 'RuntimeDefault') { - errors.push(`${prefix} pod must use RuntimeDefault seccomp`) - } - if (container.securityContext?.allowPrivilegeEscalation !== false) { - errors.push(`${prefix} must disable privilege escalation`) - } - if (container.securityContext?.readOnlyRootFilesystem !== true) { - errors.push(`${prefix} must use a read-only root filesystem`) - } - if (!container.securityContext?.capabilities?.drop?.includes('ALL')) { - errors.push(`${prefix} must drop all Linux capabilities`) - } - for (const field of ['startupProbe', 'readinessProbe', 'livenessProbe', 'resources']) { - if (container[field] === undefined) errors.push(`${prefix} must define ${field}`) - } + await validateApplicationWorkload(root, workload, errors) } return { errors, registry } From a048d6d4a45745852291706db98d69e0f404a1d9 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Mon, 27 Jul 2026 22:42:19 -0700 Subject: [PATCH 4/4] refactor: eliminate duplicated UHRP cleanup churn --- infra/uhrp-server-basic/src/index.ts | 283 ++++++++---------- infra/uhrp-server-basic/src/routes/list.ts | 25 +- infra/uhrp-server-basic/src/routes/renew.ts | 94 +++--- .../src/utils/getMetadata.ts | 25 +- .../src/utils/getPriceForFile.ts | 33 +- infra/uhrp-server-cloud-bucket/src/index.ts | 269 ++++++++--------- .../src/routes/list.ts | 6 +- .../src/routes/renew.ts | 89 +++--- .../src/serviceHealth.ts | 34 ++- .../src/utils/getMetadata.ts | 30 +- .../src/utils/getPriceForFile.ts | 33 +- 11 files changed, 412 insertions(+), 509 deletions(-) diff --git a/infra/uhrp-server-basic/src/index.ts b/infra/uhrp-server-basic/src/index.ts index 62c2e1600..37639c22d 100644 --- a/infra/uhrp-server-basic/src/index.ts +++ b/infra/uhrp-server-basic/src/index.ts @@ -29,44 +29,37 @@ import { createServiceHealth } from './serviceHealth' const SERVER_PRIVATE_KEY = process.env.SERVER_PRIVATE_KEY as string const HTTP_PORT = process.env.HTTP_PORT || 8080 -type RouteMethod = 'get' | 'put' | 'post' | 'patch' | 'delete' -const preAuthRateLimit = rateLimit( - rateLimitOptions('UHRP_PRE_AUTH_RATE_LIMIT', { windowMs: 60_000, limit: 300 }) -) +const preAuthRateLimit = rateLimit(rateLimitOptions( + 'UHRP_PRE_AUTH_RATE_LIMIT', + { windowMs: 60_000, limit: 300 } +)) -const authenticatedRateLimit = rateLimit( - rateLimitOptions( - 'UHRP_AUTHENTICATED_RATE_LIMIT', - { windowMs: 60_000, limit: 1_000 }, - { keyGenerator: authenticatedIdentityKey } - ) -) +const authenticatedRateLimit = rateLimit(rateLimitOptions( + 'UHRP_AUTHENTICATED_RATE_LIMIT', + { windowMs: 60_000, limit: 1_000 }, + { keyGenerator: authenticatedIdentityKey } +)) const app = express() const serviceHealth = createServiceHealth() - app.disable('x-powered-by') configureTrustProxy(app) app.use(securityHeaders({ environmentPrefix: 'UHRP' })) -app.use( - corsPolicy({ - environmentPrefix: 'UHRP', - methods: ['GET', 'PUT', 'POST', 'OPTIONS'] - }) -) +app.use(corsPolicy({ + environmentPrefix: 'UHRP', + methods: ['GET', 'PUT', 'POST', 'OPTIONS'] +})) app.use(concurrencyLimit('UHRP', 100)) serviceHealth.register(app) app.use(preAuthRateLimit) // Add CDN MIME type middleware before static middleware app.use(cdnMimeTypeMiddleware) app.use(express.static(path.join(__dirname, '../public'))) -app.use( - bodyparser.json({ - limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), - type: 'application/json' - }) -) +app.use(bodyparser.json({ + limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), + type: 'application/json' +})) app.use(bodyParserErrorHandler) app.use((req: Request, res: Response, next: NextFunction) => { @@ -80,148 +73,122 @@ app.use((req: Request, res: Response, next: NextFunction) => { }) // Unsecured pre-auth routes are added first -const preAuthRoutes = Object.values(routes.preAuth) -const postAuthRoutes = Object.values(routes.postAuth) +const preAuthRoutes = Object.values(routes.preAuth); +const postAuthRoutes = Object.values(routes.postAuth); // Cycle through pre-auth routes -preAuthRoutes - .filter(route => (route as any).unsecured) - .forEach(route => { - log.info( - { - operation: 'route.register', - phase: 'pre_auth_unsecured', - route_path: route.path, - route_type: route.type - }, - 'adding route' +preAuthRoutes.filter(route => (route as any).unsecured).forEach((route) => { + log.info({ operation: 'route.register', phase: 'pre_auth_unsecured', route_path: route.path, route_type: route.type }, 'adding route') + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( + route.path, + (route as any).middleware, + (route as any).func ) - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) - } else { - app[route.type as RouteMethod](route.path, (route as any).func) - } - }) + } else { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) + } +}) // Secured pre-auth routes are added next -preAuthRoutes - .filter(route => !(route as any).unsecured) - .forEach(route => { - log.info( - { - operation: 'route.register', - phase: 'pre_auth_secured', - route_path: route.path, - route_type: route.type - }, - 'adding route' +preAuthRoutes.filter(route => !(route as any).unsecured).forEach((route) => { + log.info({ operation: 'route.register', phase: 'pre_auth_secured', route_path: route.path, route_type: route.type }, 'adding route') + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( + route.path, + (route as any).middleware, + (route as any).func ) - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) - } else { - app[route.type as RouteMethod](route.path, (route as any).func) - } - }) - -// Auth is enforced from here forward -;(async () => { - const wallet = await getWallet() - const authMiddleware = createAuthMiddleware({ - wallet, - allowUnauthenticated: false - }) - - const paymentMiddleware = createPaymentMiddleware({ - wallet, - calculateRequestPrice: async req => { - if (req.url === '/upload') { - const { fileSize, retentionPeriod } = (req.body as any) || {} - if (!fileSize || !retentionPeriod) return 0 - try { - const satoshis = await getPriceForFile({ - fileSize: +fileSize, - retentionPeriod: +retentionPeriod - }) - return satoshis - } catch { - return 0 + } else { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) + } +}) + + // Auth is enforced from here forward + ; (async () => { + const wallet = await getWallet() + const authMiddleware = createAuthMiddleware({ + wallet, + allowUnauthenticated: false + }) + + const paymentMiddleware = createPaymentMiddleware({ + wallet, + calculateRequestPrice: async (req) => { + if (req.url === '/upload') { + const { fileSize, retentionPeriod } = (req.body as any) || {} + if (!fileSize || !retentionPeriod) return 0 + try { + const satoshis = await getPriceForFile({ fileSize: +fileSize, retentionPeriod: +retentionPeriod }) + return satoshis + } catch { + return 0 + } } - } - if (req.url === '/renew') { - const { uhrpUrl, additionalMinutes } = (req.body as any) || {} - if (!uhrpUrl || !additionalMinutes) return 0 - try { - const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) - const satoshis = await getPriceForFile({ - fileSize: +size, - retentionPeriod: +additionalMinutes - }) - return satoshis - } catch { - return 0 + if (req.url === '/renew') { + const { uhrpUrl, additionalMinutes } = (req.body as any) || {} + if (!uhrpUrl || !additionalMinutes) return 0 + try { + const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) + const satoshis = await getPriceForFile({ fileSize: +size, retentionPeriod: +additionalMinutes }) + return satoshis + } catch { + return 0 + } } + + return 0 + } + }) + + app.use(authMiddleware); + app.use(authenticatedRateLimit) + app.use(paymentMiddleware) + + // Secured, post-auth routes are added + postAuthRoutes.forEach((route) => { + log.info({ operation: 'route.register', phase: 'post_auth', route_path: route.path, route_type: route.type }, 'adding route') + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( + route.path, + (route as any).middleware, + (route as any).func + ) + } else { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) } + }) - return 0 - } - }) - - app.use(authMiddleware) - app.use(authenticatedRateLimit) - app.use(paymentMiddleware) - - // Secured, post-auth routes are added - postAuthRoutes.forEach(route => { - log.info( - { - operation: 'route.register', - phase: 'post_auth', - route_path: route.path, - route_type: route.type - }, - 'adding route' - ) - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) - } else { - app[route.type as RouteMethod](route.path, (route as any).func) - } - }) - - app.use((req, res) => { - log.info( - { operation: 'route.not_found', method: req.method, path: req.path }, - 'Route not found' - ) - res.status(404).json({ - status: 'error', - code: 'ERR_ROUTE_NOT_FOUND', - description: 'Route not found.' + app.use((req, res) => { + log.info({ operation: 'route.not_found', method: req.method, path: req.path }, 'Route not found') + res.status(404).json({ + status: 'error', + code: 'ERR_ROUTE_NOT_FOUND', + description: 'Route not found.' + }) }) - }) - - serviceHealth.markReady() - const server = app.listen(HTTP_PORT, () => { - const idKey = PrivateKey.fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() - log.info( - { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: idKey }, - 'UHRP Storage Server listening' - ) - }) - configureHttpServer(server, 'UHRP', { - requestTimeoutMs: 5 * 60 * 1000, - headersTimeoutMs: 15_000, - keepAliveTimeoutMs: 5_000, - socketTimeoutMs: 5 * 60 * 1000, - maxRequestsPerSocket: 1_000 - }) -})().catch(error => { - log.error( - { operation: 'bootstrap', outcome: 'error', err: error }, - 'UHRP Storage Server failed to start' - ) - process.exit(1) -}) + + serviceHealth.markReady() + const server = app.listen(HTTP_PORT, () => { + const idKey = PrivateKey + .fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() + log.info( + { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: idKey }, + 'UHRP Storage Server listening' + ) + }) + configureHttpServer(server, 'UHRP', { + requestTimeoutMs: 5 * 60 * 1000, + headersTimeoutMs: 15_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 5 * 60 * 1000, + maxRequestsPerSocket: 1_000 + }) + + })().catch((error) => { + log.error({ operation: 'bootstrap', outcome: 'error', err: error }, 'UHRP Storage Server failed to start') + process.exit(1) + }); diff --git a/infra/uhrp-server-basic/src/routes/list.ts b/infra/uhrp-server-basic/src/routes/list.ts index f2993f21d..4c8c21a17 100644 --- a/infra/uhrp-server-basic/src/routes/list.ts +++ b/infra/uhrp-server-basic/src/routes/list.ts @@ -25,12 +25,12 @@ interface ListResponse { const listHandler = async (req: ListRequest, res: Response) => { if (!req.auth.identityKey || req.auth.identityKey === 'unknown') { - return res.status(400).json({ - status: 'error', - code: 'ERR_MISSING_IDENTITY_KEY', - description: 'Missing authfetch identityKey.' - }) - } + return res.status(400).json({ + status: 'error', + code: 'ERR_MISSING_IDENTITY_KEY', + description: 'Missing authfetch identityKey.' + }) + } try { const identityKey = req.auth.identityKey if (!identityKey) { @@ -66,7 +66,7 @@ const listHandler = async (req: ListRequest, res: Response) => { : '' const expiryTime = expiryTimeTag - ? Number.parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) + ? parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) : 0 if (Date.now() > expiryTime * 1000) { @@ -84,10 +84,7 @@ const listHandler = async (req: ListRequest, res: Response) => { uploads: result }) } catch (error) { - log.error( - { operation: 'list.handle', outcome: 'error', err: error }, - 'Error listing advertisements' - ) + log.error({ operation: 'list.handle', outcome: 'error', err: error }, 'Error listing advertisements') return res.status(500).json({ status: 'error', code: 'ERR_LIST', @@ -99,13 +96,13 @@ const listHandler = async (req: ListRequest, res: Response) => { export default { type: 'get', path: '/list', - summary: - "Lists all UHRP files (advertisements) matching the user's identityKey in transaction tags.", + summary: 'Lists all UHRP files (advertisements) matching the user\'s identityKey in transaction tags.', parameters: {}, exampleResponse: { status: 'success', uploads: [ { + uhrpUrl: 'uhrp://abcd1234...', expiryTime: 1691234567 } @@ -113,4 +110,4 @@ export default { }, errors: ['ERR_LIST'], func: listHandler -} +} \ No newline at end of file diff --git a/infra/uhrp-server-basic/src/routes/renew.ts b/infra/uhrp-server-basic/src/routes/renew.ts index d00b0011d..6e1565941 100644 --- a/infra/uhrp-server-basic/src/routes/renew.ts +++ b/infra/uhrp-server-basic/src/routes/renew.ts @@ -28,25 +28,17 @@ interface RenewResponse { description?: string } -const latestAdvertisement = (outputs: T[]): T | undefined => { - let latest: T | undefined - let latestExpiry = 0 - for (const output of outputs) { - const expiryTag = output.tags?.find(tag => tag.startsWith('expiry_time_')) - if (expiryTag == null) continue - const expiry = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 - if (expiry > latestExpiry) { - latestExpiry = expiry - latest = output - } - } - return latest -} - const renewHandler = async (req: RenewRequest, res: Response) => { + if (!req.auth.identityKey || req.auth.identityKey === 'unknown') { + return res.status(400).json({ + status: 'error', + code: 'ERR_MISSING_IDENTITY_KEY', + description: 'Missing authfetch identityKey.' + }) + } try { const { identityKey } = req.auth - if (!identityKey || identityKey === 'unknown') { + if (!identityKey) { return res.status(400).json({ status: 'error', code: 'ERR_MISSING_IDENTITY_KEY', @@ -76,9 +68,9 @@ const renewHandler = async (req: RenewRequest, res: Response) => } = await getMetadata(uhrpUrl, identityKey, limit, offset) // Convert to MS to create an ISO string - const newExpiryTimeSeconds = prevExpiryTime + additionalMinutes * 60 + const newExpiryTimeSeconds = prevExpiryTime + (additionalMinutes * 60) - const fileSizeNum = Number.parseInt(size, 10) || 0 + const fileSizeNum = parseInt(size, 10) || 0 let amount = 0 if (fileSizeNum > 0) { amount = await getPriceForFile({ @@ -89,12 +81,9 @@ const renewHandler = async (req: RenewRequest, res: Response) => // When multiple advertisements match, renew the one with the farthest expiry. const wallet = await getWallet() - const { outputs, BEEF } = await wallet.listOutputs({ + const { outputs, BEEF, } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [ - `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, - `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}` - ], + tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`], tagQueryMode: 'all', includeTags: true, include: 'entire transactions', @@ -110,7 +99,22 @@ const renewHandler = async (req: RenewRequest, res: Response) => }) } - const prevAdvertisement = latestAdvertisement(outputs) + // Finding the maxpiry file with the same url + let prevAdvertisement + // Farthest expiration time given in seconds + let maxpiry = 0 + for (const out of outputs) { + if (!out.tags) continue + const expiryTag = out.tags.find(t => t.startsWith('expiry_time_')) + if (!expiryTag) continue + + const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + + if (expiryNum > maxpiry) { + maxpiry = expiryNum + prevAdvertisement = out + } + } if (!prevAdvertisement || !BEEF) { return res.status(404).json({ @@ -149,30 +153,24 @@ const renewHandler = async (req: RenewRequest, res: Response) => const uploaderTag = prevAdvertisement.tags.find(t => t.startsWith('uploader_identity_key_')) if (uploaderTag) newTags.push(uploaderTag) } - newTags.push( - `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, - `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`, - `expiry_time_${newExpiryTimeSeconds}` - ) + newTags.push(`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`) + newTags.push(`object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`) + newTags.push(`expiry_time_${newExpiryTimeSeconds}`) const { signableTransaction } = await wallet.createAction({ inputBEEF: BEEF, - inputs: [ - { - outpoint: prevAdvertisement.outpoint, - unlockingScriptLength: 74, - inputDescription: 'Redeeming old advertisement' - } - ], - outputs: [ - { - lockingScript: newLockingScript.toHex(), - satoshis: 1, - basket: 'uhrp advertisements', - outputDescription: 'UHRP advertisement token (renewed)', - tags: newTags - } - ], + inputs: [{ + outpoint: prevAdvertisement.outpoint, + unlockingScriptLength: 74, + inputDescription: 'Redeeming old advertisement' + }], + outputs: [{ + lockingScript: newLockingScript.toHex(), + satoshis: 1, + basket: 'uhrp advertisements', + outputDescription: 'UHRP advertisement token (renewed)', + tags: newTags + }], description: `Renew advertisement for uhrpUrl ${uhrpUrl}`, options: { randomizeOutputs: false @@ -192,7 +190,8 @@ const renewHandler = async (req: RenewRequest, res: Response) => const unlockingScript = await unlocker.sign(partialTx, 0) const { tx, txid } = await wallet.signAction({ reference: signableTransaction.reference, - spends: { + spends: + { 0: { unlockingScript: unlockingScript.toHex() } @@ -230,8 +229,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => export default { type: 'post', path: '/renew', - summary: - 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', + summary: 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', parameters: { uhrpUrl: 'The UHRP URL (e.g. "uhrp://somehash")', additionalMinutes: 'Number of minutes to extend' diff --git a/infra/uhrp-server-basic/src/utils/getMetadata.ts b/infra/uhrp-server-basic/src/utils/getMetadata.ts index 55b60af46..15fa5bdee 100644 --- a/infra/uhrp-server-basic/src/utils/getMetadata.ts +++ b/infra/uhrp-server-basic/src/utils/getMetadata.ts @@ -1,12 +1,13 @@ import { getWallet } from './walletSingleton' import { Utils } from '@bsv/sdk' + interface FileMetadata { objectIdentifier: string name: string size: string contentType: string - expiryTime: number // minutes since the Unix epoch + expiryTime: number // minutes since the Unix epoch } /** @@ -17,19 +18,11 @@ interface FileMetadata { * @returns {Promise} An object containing file info. * @throws If no matching advertisement is found or GCS metadata fails. */ -export async function getMetadata( - uhrpUrl: string, - uploaderIdentityKey: string, - limit?: number, - offset?: number -): Promise { +export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, limit?: number, offset?: number): Promise { const wallet = await getWallet() const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [ - `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, - `uploader_identity_key_${uploaderIdentityKey}` - ], + tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `uploader_identity_key_${uploaderIdentityKey}`], tagQueryMode: 'all', includeTags: true, limit: limit ?? 200, @@ -49,13 +42,11 @@ export async function getMetadata( const contentTypeTag = out.tags.find(t => t.startsWith('content_type_')) if (!objectIdTag || !expiryTag || !nameTag || !sizeTag || !contentTypeTag) continue - const expiryNum = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 if (expiryNum > maxpiry) { maxpiry = expiryNum - objectIdentifier = Utils.toUTF8( - Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex') - ) + objectIdentifier = Utils.toUTF8(Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex')) name = nameTag size = sizeTag contentType = contentTypeTag @@ -63,9 +54,7 @@ export async function getMetadata( } if (!objectIdentifier || !name || !size || !contentType) { - throw new Error( - `No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}` - ) + throw new Error(`No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}`) } if (Date.now() > maxpiry * 1000) { diff --git a/infra/uhrp-server-basic/src/utils/getPriceForFile.ts b/infra/uhrp-server-basic/src/utils/getPriceForFile.ts index 416863490..9ce3421e7 100644 --- a/infra/uhrp-server-basic/src/utils/getPriceForFile.ts +++ b/infra/uhrp-server-basic/src/utils/getPriceForFile.ts @@ -14,17 +14,14 @@ interface PriceCalculationParams { * @param {PriceCalculationParams} params - Parameters for price calculation. * @returns {Promise} - The price in satoshis. */ -const getPriceForFile = async ({ - retentionPeriod, - fileSize -}: PriceCalculationParams): Promise => { +const getPriceForFile = async ({ retentionPeriod, fileSize }: PriceCalculationParams): Promise => { if (!PRICE_PER_GB_MO) { throw new Error('PRICE_PER_GB_MO is undefined') } - const pricePerGBMonth = Number.parseFloat(PRICE_PER_GB_MO) - if (Number.isNaN(pricePerGBMonth)) { - throw new TypeError('PRICE_PER_GB_MO must be a valid number') + const pricePerGBMonth = parseFloat(PRICE_PER_GB_MO) + if (isNaN(pricePerGBMonth)) { + throw new Error('PRICE_PER_GB_MO must be a valid number') } // File size is in bytes, convert to gigabytes @@ -39,30 +36,24 @@ const getPriceForFile = async ({ // Get the exchange rate let exchangeRate: number try { - const { data } = await axios.get('https://api.whatsonchain.com/v1/bsv/main/exchangerate') - if ( - data == null || - typeof data !== 'object' || - typeof data.rate !== 'number' || - !Number.isFinite(data.rate) - ) { - throw new TypeError('Invalid rate response') + const { data } = await axios.get( + 'https://api.whatsonchain.com/v1/bsv/main/exchangerate' + ) + if (typeof data !== 'object' || isNaN(data.rate)) { + throw new Error('Invalid rate response') } exchangeRate = data.rate } catch (e) { exchangeRate = 30 - log.error( - { operation: 'exchange_rate.fetch', outcome: 'error', fallback_rate: 30, err: e }, - 'Exchange rate failed, using fallback rate' - ) + log.error({ operation: 'exchange_rate.fetch', outcome: 'error', fallback_rate: 30, err: e }, 'Exchange rate failed, using fallback rate') } // Exchange rate is in BSV, convert to satoshis const exchangeRateInSatoshis = 1 / (exchangeRate / 100000000) // Account for server overhead in our prices, so there is a minimum of 10 satoshis - const satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)) + let satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)); return satPrice } -export default getPriceForFile +export default getPriceForFile \ No newline at end of file diff --git a/infra/uhrp-server-cloud-bucket/src/index.ts b/infra/uhrp-server-cloud-bucket/src/index.ts index 2080c547a..f32bb7d1a 100644 --- a/infra/uhrp-server-cloud-bucket/src/index.ts +++ b/infra/uhrp-server-cloud-bucket/src/index.ts @@ -28,51 +28,41 @@ import { createServiceHealth } from './serviceHealth' const SERVER_PRIVATE_KEY = process.env.SERVER_PRIVATE_KEY as string const HTTP_PORT = process.env.HTTP_PORT || 8080 const NODE_ENV = process.env.NODE_ENV || 'development' -type RouteMethod = 'get' | 'put' | 'post' | 'patch' | 'delete' -const preAuthRateLimit = rateLimit( - rateLimitOptions('UHRP_PRE_AUTH_RATE_LIMIT', { windowMs: 60_000, limit: 300 }) -) +const preAuthRateLimit = rateLimit(rateLimitOptions( + 'UHRP_PRE_AUTH_RATE_LIMIT', + { windowMs: 60_000, limit: 300 } +)) -const authenticatedRateLimit = rateLimit( - rateLimitOptions( - 'UHRP_AUTHENTICATED_RATE_LIMIT', - { windowMs: 60_000, limit: 1_000 }, - { keyGenerator: authenticatedIdentityKey } - ) -) +const authenticatedRateLimit = rateLimit(rateLimitOptions( + 'UHRP_AUTHENTICATED_RATE_LIMIT', + { windowMs: 60_000, limit: 1_000 }, + { keyGenerator: authenticatedIdentityKey } +)) const app = express() const serviceHealth = createServiceHealth() - app.disable('x-powered-by') configureTrustProxy(app) app.use(securityHeaders({ environmentPrefix: 'UHRP' })) -app.use( - corsPolicy({ - environmentPrefix: 'UHRP', - methods: ['GET', 'POST', 'OPTIONS'] - }) -) +app.use(corsPolicy({ + environmentPrefix: 'UHRP', + methods: ['GET', 'POST', 'OPTIONS'] +})) app.use(concurrencyLimit('UHRP', 200)) serviceHealth.register(app) app.use(preAuthRateLimit) -app.use( - bodyparser.json({ - limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), - type: 'application/json' - }) -) +app.use(bodyparser.json({ + limit: readBodyLimitBytes('UHRP_JSON', 256 * 1024), + type: 'application/json' +})) app.use(bodyParserErrorHandler) app.use((req: Request, res: Response, next: NextFunction) => { log.info({ operation: 'request.in', method: req.method, url: req.url }, 'Incoming request') const originalJson = res.json.bind(res) res.json = (json: any) => { - log.info( - { operation: 'response.json', method: req.method, url: req.url }, - 'Outgoing JSON response' - ) + log.info({ operation: 'response.json', method: req.method, url: req.url }, 'Outgoing JSON response') return originalJson(json) } next() @@ -81,24 +71,23 @@ app.use((req: Request, res: Response, next: NextFunction) => { app.use(express.static('public')) // Unsecured pre-auth routes are added first -const preAuthRoutes = Object.values(routes.preAuth) -const postAuthRoutes = Object.values(routes.postAuth) +const preAuthRoutes = Object.values(routes.preAuth); +const postAuthRoutes = Object.values(routes.postAuth); // Cycle through pre-auth routes -preAuthRoutes - .filter(route => (route as any).unsecured) - .forEach(route => { - log.info( - { operation: 'route.register', phase: 'pre_auth', secured: false, route_path: route.path }, - 'Registering route' +preAuthRoutes.filter(route => (route as any).unsecured).forEach((route) => { + log.info({ operation: 'route.register', phase: 'pre_auth', secured: false, route_path: route.path }, 'Registering route') + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( + route.path, + (route as any).middleware, + (route as any).func ) - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) - } else { - app[route.type as RouteMethod](route.path, (route as any).func) - } - }) + } else { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) + } +}) // This ensures that HTTPS is used for uploads app.use((req: Request, res: Response, next: NextFunction) => { @@ -110,113 +99,107 @@ app.use((req: Request, res: Response, next: NextFunction) => { }) } next() -}) +}); // Secured pre-auth routes are added after the HTTPS redirect -preAuthRoutes - .filter(route => !(route as any).unsecured) - .forEach(route => { - log.info( - { operation: 'route.register', phase: 'pre_auth', secured: true, route_path: route.path }, - 'Registering route' +preAuthRoutes.filter(route => !(route as any).unsecured).forEach((route) => { + log.info({ operation: 'route.register', phase: 'pre_auth', secured: true, route_path: route.path }, 'Registering route') + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( + route.path, + (route as any).middleware, + (route as any).func ) - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) - } else { - app[route.type as RouteMethod](route.path, (route as any).func) - } - }) - -// Auth is enforced from here forward -;(async () => { - const wallet = await getWallet() - const authMiddleware = createAuthMiddleware({ - wallet, - allowUnauthenticated: false - }) - - const paymentMiddleware = createPaymentMiddleware({ - wallet, - calculateRequestPrice: async req => { - if (req.url === '/upload') { - const { fileSize, retentionPeriod } = (req.body as any) || {} - if (!fileSize || !retentionPeriod) return 0 - try { - const satoshis = await getPriceForFile({ - fileSize: +fileSize, - retentionPeriod: +retentionPeriod - }) - return satoshis - } catch { - return 0 + } else { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) + } +}) + + // Auth is enforced from here forward + ; (async () => { + const wallet = await getWallet() + const authMiddleware = createAuthMiddleware({ + wallet, + allowUnauthenticated: false + }) + + const paymentMiddleware = createPaymentMiddleware({ + wallet, + calculateRequestPrice: async (req) => { + + if (req.url === '/upload') { + const { fileSize, retentionPeriod } = (req.body as any) || {} + if (!fileSize || !retentionPeriod) return 0 + try { + const satoshis = await getPriceForFile({ fileSize: +fileSize, retentionPeriod: +retentionPeriod }) + return satoshis + } catch { + return 0 + } } - } - if (req.url === '/renew') { - const { uhrpUrl, additionalMinutes } = (req.body as any) || {} - if (!uhrpUrl || !additionalMinutes) return 0 - try { - const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) - const satoshis = await getPriceForFile({ - fileSize: +size, - retentionPeriod: +additionalMinutes - }) - return satoshis - } catch { - return 0 + if (req.url === '/renew') { + const { uhrpUrl, additionalMinutes } = (req.body as any) || {} + if (!uhrpUrl || !additionalMinutes) return 0 + try { + const { size } = await getMetadata(uhrpUrl, (req as any).auth.identityKey) + const satoshis = await getPriceForFile({ fileSize: +size, retentionPeriod: +additionalMinutes }) + return satoshis + } catch { + return 0 + } } + + return 0 } + }) - return 0 - } - }) + app.use(authMiddleware); + app.use(authenticatedRateLimit) + app.use(paymentMiddleware) + + // Secured, post-auth routes are added + postAuthRoutes.forEach((route) => { + log.info({ operation: 'route.register', phase: 'post_auth', secured: true, route_path: route.path }, 'Registering route') + // If we need middleware for a route, attach it + if ((route as any).middleware) { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete']( + route.path, + (route as any).middleware, + (route as any).func + ) + } else { + app[route.type as 'get' | 'put' | 'post' | 'patch' | 'delete'](route.path, (route as any).func) + } + }) - app.use(authMiddleware) - app.use(authenticatedRateLimit) - app.use(paymentMiddleware) + app.use((req, res) => { + log.warn({ operation: 'route.not_found', method: req.method, url: req.url }, 'Route not found') + res.status(404).json({ + status: 'error', + code: 'ERR_ROUTE_NOT_FOUND', + description: 'Route not found.' + }) + }) - // Secured, post-auth routes are added - postAuthRoutes.forEach(route => { - log.info( - { operation: 'route.register', phase: 'post_auth', secured: true, route_path: route.path }, - 'Registering route' - ) - // If we need middleware for a route, attach it - if ((route as any).middleware) { - app[route.type as RouteMethod](route.path, (route as any).middleware, (route as any).func) - } else { - app[route.type as RouteMethod](route.path, (route as any).func) - } - }) - - app.use((req, res) => { - log.warn({ operation: 'route.not_found', method: req.method, url: req.url }, 'Route not found') - res.status(404).json({ - status: 'error', - code: 'ERR_ROUTE_NOT_FOUND', - description: 'Route not found.' + serviceHealth.markReady() + const server = app.listen(HTTP_PORT, () => { + const identityKey = PrivateKey + .fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() + log.info( + { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: identityKey }, + 'UHRP Storage Server listening' + ) }) - }) - - serviceHealth.markReady() - const server = app.listen(HTTP_PORT, () => { - const identityKey = PrivateKey.fromString(SERVER_PRIVATE_KEY).toPublicKey().toString() - log.info( - { operation: 'listen', outcome: 'ok', port: HTTP_PORT, identity_key: identityKey }, - 'UHRP Storage Server listening' - ) - }) - configureHttpServer(server, 'UHRP', { - requestTimeoutMs: 60_000, - headersTimeoutMs: 15_000, - keepAliveTimeoutMs: 5_000, - socketTimeoutMs: 60_000, - maxRequestsPerSocket: 1_000 - }) -})().catch(error => { - log.error( - { operation: 'bootstrap', outcome: 'error', err: error }, - 'UHRP Storage Server failed to start' - ) - process.exit(1) -}) + configureHttpServer(server, 'UHRP', { + requestTimeoutMs: 60_000, + headersTimeoutMs: 15_000, + keepAliveTimeoutMs: 5_000, + socketTimeoutMs: 60_000, + maxRequestsPerSocket: 1_000 + }) + + })().catch((error) => { + log.error({ operation: 'bootstrap', outcome: 'error', err: error }, 'UHRP Storage Server failed to start') + process.exit(1) + }); diff --git a/infra/uhrp-server-cloud-bucket/src/routes/list.ts b/infra/uhrp-server-cloud-bucket/src/routes/list.ts index 4766c2195..c38bf6f1d 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/list.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/list.ts @@ -59,7 +59,7 @@ const listHandler = async (req: ListRequest, res: Response) => { : '' const expiryTime = expiryTimeTag - ? Number.parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) + ? parseInt(expiryTimeTag.substring('expiry_time_'.length), 10) : 0 if (Date.now() > expiryTime * 1000) { @@ -89,13 +89,13 @@ const listHandler = async (req: ListRequest, res: Response) => { export default { type: 'get', path: '/list', - summary: - "Lists all UHRP files (advertisements) matching the user's identityKey in transaction tags.", + summary: 'Lists all UHRP files (advertisements) matching the user\'s identityKey in transaction tags.', parameters: {}, exampleResponse: { status: 'success', uploads: [ { + uhrpUrl: 'uhrp://abcd1234...', expiryTime: 1691234567 } diff --git a/infra/uhrp-server-cloud-bucket/src/routes/renew.ts b/infra/uhrp-server-cloud-bucket/src/routes/renew.ts index 842edce09..928e4b40d 100644 --- a/infra/uhrp-server-cloud-bucket/src/routes/renew.ts +++ b/infra/uhrp-server-cloud-bucket/src/routes/renew.ts @@ -31,21 +31,6 @@ interface RenewResponse { description?: string } -const latestAdvertisement = (outputs: T[]): T | undefined => { - let latest: T | undefined - let latestExpiry = 0 - for (const output of outputs) { - const expiryTag = output.tags?.find(tag => tag.startsWith('expiry_time_')) - if (expiryTag == null) continue - const expiry = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 - if (expiry > latestExpiry) { - latestExpiry = expiry - latest = output - } - } - return latest -} - const renewHandler = async (req: RenewRequest, res: Response) => { try { const { identityKey } = req.auth @@ -79,10 +64,10 @@ const renewHandler = async (req: RenewRequest, res: Response) => } = await getMetadata(uhrpUrl, identityKey, limit, offset) // Convert to MS to create an ISO string - const newExpiryTimeSeconds = prevExpiryTime + additionalMinutes * 60 + const newExpiryTimeSeconds = prevExpiryTime + (additionalMinutes * 60) const newCustomTimeIso = new Date(newExpiryTimeSeconds * 1000).toISOString() - const fileSizeNum = Number.parseInt(size, 10) || 0 + const fileSizeNum = parseInt(size, 10) || 0 let amount = 0 if (fileSizeNum > 0) { amount = await getPriceForFile({ @@ -93,12 +78,9 @@ const renewHandler = async (req: RenewRequest, res: Response) => // When multiple advertisements match, renew the one with the farthest expiry. const wallet = await getWallet() - const { outputs, BEEF } = await wallet.listOutputs({ + const { outputs, BEEF, } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [ - `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, - `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}` - ], + tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`], tagQueryMode: 'all', includeTags: true, include: 'entire transactions', @@ -114,7 +96,22 @@ const renewHandler = async (req: RenewRequest, res: Response) => }) } - const prevAdvertisement = latestAdvertisement(outputs) + // Finding the maxpiry file with the same url + let prevAdvertisement + // Farthest expiration time given in seconds + let maxpiry = 0 + for (const out of outputs) { + if (!out.tags) continue + const expiryTag = out.tags.find(t => t.startsWith('expiry_time_')) + if (!expiryTag) continue + + const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + + if (expiryNum > maxpiry) { + maxpiry = expiryNum + prevAdvertisement = out + } + } if (!prevAdvertisement || !BEEF) { return res.status(404).json({ @@ -153,30 +150,24 @@ const renewHandler = async (req: RenewRequest, res: Response) => const uploaderTag = prevAdvertisement.tags.find(t => t.startsWith('uploader_identity_key_')) if (uploaderTag) newTags.push(uploaderTag) } - newTags.push( - `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, - `object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`, - `expiry_time_${newExpiryTimeSeconds}` - ) + newTags.push(`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`) + newTags.push(`object_identifier_${Utils.toHex(Utils.toArray(objectIdentifier, 'utf8'))}`) + newTags.push(`expiry_time_${newExpiryTimeSeconds}`) const { signableTransaction } = await wallet.createAction({ inputBEEF: BEEF, - inputs: [ - { - outpoint: prevAdvertisement.outpoint, - unlockingScriptLength: 74, - inputDescription: 'Redeeming old advertisement' - } - ], - outputs: [ - { - lockingScript: newLockingScript.toHex(), - satoshis: 1, - basket: 'uhrp advertisements', - outputDescription: 'UHRP advertisement token (renewed)', - tags: newTags - } - ], + inputs: [{ + outpoint: prevAdvertisement.outpoint, + unlockingScriptLength: 74, + inputDescription: 'Redeeming old advertisement' + }], + outputs: [{ + lockingScript: newLockingScript.toHex(), + satoshis: 1, + basket: 'uhrp advertisements', + outputDescription: 'UHRP advertisement token (renewed)', + tags: newTags + }], description: `Renew advertisement for uhrpUrl ${uhrpUrl}`, options: { randomizeOutputs: false @@ -196,7 +187,8 @@ const renewHandler = async (req: RenewRequest, res: Response) => const unlockingScript = await unlocker.sign(partialTx, 0) const { tx, txid } = await wallet.signAction({ reference: signableTransaction.reference, - spends: { + spends: + { 0: { unlockingScript: unlockingScript.toHex() } @@ -216,9 +208,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => await broadcaster.broadcast(Transaction.fromAtomicBEEF(tx)) // Setting the new expiry time in the actual database - await storage - .bucket(GCP_BUCKET_NAME) - .file(`cdn/${objectIdentifier}`) + await storage.bucket(GCP_BUCKET_NAME).file(`cdn/${objectIdentifier}`) .setMetadata({ customTime: newCustomTimeIso }) return res.status(200).json({ @@ -240,8 +230,7 @@ const renewHandler = async (req: RenewRequest, res: Response) => export default { type: 'post', path: '/renew', - summary: - 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', + summary: 'Renews storage time by adding additionalMinutes to the GCS customTime of a file found by uhrpUrl.', parameters: { uhrpUrl: 'The UHRP URL (e.g. "uhrp://somehash")', additionalMinutes: 'Number of minutes to extend' diff --git a/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts b/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts index 3543a6225..bc9874f59 100644 --- a/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts +++ b/infra/uhrp-server-cloud-bucket/src/serviceHealth.ts @@ -5,20 +5,26 @@ export interface ServiceHealth { register: (app: Express) => void } -export const createServiceHealth = (): ServiceHealth => { - let ready = false +class CloudServiceHealth implements ServiceHealth { + private ready = false - return { - markReady: () => { - ready = true - }, - register: app => { - app.get('/health', (_req: Request, res: Response) => { - res.status(200).json({ status: 'ok', live: true }) - }) - app.get('/ready', (_req: Request, res: Response) => { - res.status(ready ? 200 : 503).json({ status: ready ? 'ready' : 'starting', ready }) - }) - } + public readonly markReady = (): void => { + this.ready = true + } + + public readonly register = (app: Express): void => { + app.get('/health', this.reportLiveness) + app.get('/ready', this.reportReadiness) + } + + private readonly reportLiveness = (_req: Request, res: Response): void => { + res.status(200).json({ status: 'ok', live: true }) + } + + private readonly reportReadiness = (_req: Request, res: Response): void => { + const status = this.ready ? 'ready' : 'starting' + res.status(this.ready ? 200 : 503).json({ status, ready: this.ready }) } } + +export const createServiceHealth = (): ServiceHealth => new CloudServiceHealth() diff --git a/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts b/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts index 70df832c4..19788d43a 100644 --- a/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts +++ b/infra/uhrp-server-cloud-bucket/src/utils/getMetadata.ts @@ -11,7 +11,7 @@ interface FileMetadata { name: string size: string contentType: string - expiryTime: number // minutes since the Unix epoch + expiryTime: number // minutes since the Unix epoch } /** @@ -22,19 +22,11 @@ interface FileMetadata { * @returns {Promise} An object containing file info. * @throws If no matching advertisement is found or GCS metadata fails. */ -export async function getMetadata( - uhrpUrl: string, - uploaderIdentityKey: string, - limit?: number, - offset?: number -): Promise { +export async function getMetadata(uhrpUrl: string, uploaderIdentityKey: string, limit?: number, offset?: number): Promise { const wallet = await getWallet() const { outputs } = await wallet.listOutputs({ basket: 'uhrp advertisements', - tags: [ - `uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, - `uploader_identity_key_${uploaderIdentityKey}` - ], + tags: [`uhrp_url_${Utils.toHex(Utils.toArray(uhrpUrl, 'utf8'))}`, `uploader_identity_key_${uploaderIdentityKey}`], tagQueryMode: 'all', includeTags: true, limit: limit ?? 200, @@ -51,20 +43,16 @@ export async function getMetadata( const expiryTag = out.tags.find(t => t.startsWith('expiry_time_')) if (!objectIdTag || !expiryTag) continue - const expiryNum = Number.parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 + const expiryNum = parseInt(expiryTag.substring('expiry_time_'.length), 10) || 0 if (expiryNum > maxpiry) { maxpiry = expiryNum - objectIdentifier = Utils.toUTF8( - Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex') - ) + objectIdentifier = Utils.toUTF8(Utils.toArray(objectIdTag.substring('object_identifier_'.length), 'hex')) } } if (!objectIdentifier) { - throw new Error( - `No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}` - ) + throw new Error(`No advertisement found for uhrpUrl: ${uhrpUrl} uploaderIdentityKey: ${uploaderIdentityKey}`) } if (Date.now() > maxpiry * 1000) { @@ -75,7 +63,11 @@ export async function getMetadata( const file = storage.bucket(GCP_BUCKET_NAME!).file(`cdn/${objectIdentifier}`) const [gcsMetadata] = await file.getMetadata() - const { name = '', size, contentType = '' } = gcsMetadata + const { + name = '', + size, + contentType = '', + } = gcsMetadata return { objectIdentifier, diff --git a/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts b/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts index a2c66c9dd..423adf68c 100644 --- a/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts +++ b/infra/uhrp-server-cloud-bucket/src/utils/getPriceForFile.ts @@ -14,17 +14,14 @@ interface PriceCalculationParams { * @param {PriceCalculationParams} params - Parameters for price calculation. * @returns {Promise} - The price in satoshis. */ -const getPriceForFile = async ({ - retentionPeriod, - fileSize -}: PriceCalculationParams): Promise => { +const getPriceForFile = async ({ retentionPeriod, fileSize }: PriceCalculationParams): Promise => { if (!PRICE_PER_GB_MO) { throw new Error('PRICE_PER_GB_MO is undefined') } - const pricePerGBMonth = Number.parseFloat(PRICE_PER_GB_MO) - if (Number.isNaN(pricePerGBMonth)) { - throw new TypeError('PRICE_PER_GB_MO must be a valid number') + const pricePerGBMonth = parseFloat(PRICE_PER_GB_MO) + if (isNaN(pricePerGBMonth)) { + throw new Error('PRICE_PER_GB_MO must be a valid number') } // File size is in bytes, convert to gigabytes @@ -39,30 +36,24 @@ const getPriceForFile = async ({ // Get the exchange rate let exchangeRate: number try { - const { data } = await axios.get('https://api.whatsonchain.com/v1/bsv/main/exchangerate') - if ( - data == null || - typeof data !== 'object' || - typeof data.rate !== 'number' || - !Number.isFinite(data.rate) - ) { - throw new TypeError('Invalid rate response') + const { data } = await axios.get( + 'https://api.whatsonchain.com/v1/bsv/main/exchangerate' + ) + if (typeof data !== 'object' || isNaN(data.rate)) { + throw new Error('Invalid rate response') } exchangeRate = data.rate } catch (e) { exchangeRate = 30 - log.error( - { operation: 'price.exchange_rate', outcome: 'error', fallback_rate: 30, err: e }, - 'Exchange rate failed, using fallback rate' - ) + log.error({ operation: 'price.exchange_rate', outcome: 'error', fallback_rate: 30, err: e }, 'Exchange rate failed, using fallback rate') } // Exchange rate is in BSV, convert to satoshis const exchangeRateInSatoshis = 1 / (exchangeRate / 100000000) // Account for server overhead in our prices, so there is a minimum of 10 satoshis - const satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)) + let satPrice = Math.max(10, Math.floor(usdPrice * exchangeRateInSatoshis)); return satPrice } -export default getPriceForFile +export default getPriceForFile \ No newline at end of file