Fix the three pages that shipped broken, and the checks that let them

.github/workflows/universe-ci.yml

@@ -52,6 +52,9 @@ jobs:
       - name: Gateway routing
         run: node --test scripts/universe/gateway.test.mjs
 
+      - name: The gateway bridges an upstream restart rather than failing
+        run: node --test scripts/universe/gateway-restart.test.mjs
+
   backend:
     name: Backend build and test
     runs-on: [self-hosted, linux-ultra]
@@ -83,6 +86,13 @@ jobs:
         working-directory: backend
         run: npm run test:ci
 
+      # The failure that shipped was a configuration one: the migrations never
+      # ran against the engine production uses, so nothing caught the syntax it
+      # rejects. These run the real backend against a real database.
+      - name: Migrations and repositories against a real database
+        working-directory: backend
+        run: npm run test:integration
+
   frontend:
     name: Frontend build
     runs-on: [self-hosted, linux-ultra]
@@ -127,17 +137,34 @@ jobs:
         working-directory: scripts/universe/visual-qa
         run: npm ci
 
+      - name: The unfinished-page gate fires on the states that shipped
+        working-directory: scripts/universe/visual-qa
+        run: node --test progress-gate.test.mjs
+
+      # A fixed port here once measured another job's build: two runners share
+      # a host, the second gateway crashed on the taken port behind "&", the
+      # probe happily reached the first job's server, and the run died halfway
+      # when that other job ended. Each run now serves on its own free port and
+      # refuses to continue unless the process answering is the one it started.
       - name: Serve the built frontend
         run: |
-          UNIVERSE_GATEWAY_PORT=8123           UNIVERSE_GATEWAY_ROOT=frontend/dist/mempool/browser           node scripts/universe/gateway.mjs &
+          GATEWAY_PORT=$(node -e "const s=require('net').createServer();s.listen(0,'127.0.0.1',()=>{console.log(s.address().port);s.close()})")
+          echo "GATEWAY_PORT=$GATEWAY_PORT" >> "$GITHUB_ENV"
+          UNIVERSE_GATEWAY_PORT=$GATEWAY_PORT           UNIVERSE_GATEWAY_ROOT=frontend/dist/mempool/browser           node scripts/universe/gateway.mjs &
+          GATEWAY_PID=$!
           for i in $(seq 1 30); do
-            curl -fsS -o /dev/null http://127.0.0.1:8123/ && break
+            curl -fsS -o /dev/null "http://127.0.0.1:$GATEWAY_PORT/" && break
             sleep 1
           done
+          kill -0 "$GATEWAY_PID"
+          curl -fsS "http://127.0.0.1:$GATEWAY_PORT/__gateway/health"
 
-      - name: Measured contrast, accessibility, overflow, and screenshots
+      # The harness now fails the run when a page never finishes loading, when a
+      # chart panel draws nothing, or when a failure state says nothing about
+      # why. Those three states passed every other gate here and shipped.
+      - name: Measured contrast, accessibility, overflow, unfinished pages, and screenshots
         working-directory: scripts/universe/visual-qa
-        run: node capture.mjs --base=http://127.0.0.1:8123 --out=./artifacts
+        run: node capture.mjs --base=http://127.0.0.1:$GATEWAY_PORT --out=./artifacts
 
       - name: Keep the contrast report and the route screenshots
         if: always()

.github/workflows/universe-production-smoke.yml

@@ -0,0 +1,44 @@
+name: Universe production smoke
+
+# Checks the live origin with nothing mocked.
+#
+# The rest of the suite answers every request from fixtures, which is what let
+# a release ship with a Charts page and a Mining dashboard in front of routes
+# that answered 404. Fixtures prove how a page renders given data; only this
+# proves the deployment can produce any.
+
+on:
+  # After a release, and on a schedule so a dependency that fails later is
+  # noticed without anyone deploying.
+  workflow_dispatch:
+    inputs:
+      origin:
+        description: Origin to check
+        required: false
+        default: https://explorer.bitcoinuniverse.io
+  schedule:
+    - cron: '17 * * * *'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: universe-production-smoke
+  cancel-in-progress: false
+
+jobs:
+  smoke:
+    name: Live origin answers with real content
+    runs-on: [self-hosted, linux-ultra]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Setup Node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '24.19.0'
+
+      - name: Every advertised feature has routes, data, and a matching release
+        run: node scripts/universe/synthetic-check.mjs "${{ github.event.inputs.origin || 'https://explorer.bitcoinuniverse.io' }}"

.gitignore

@@ -15,4 +15,6 @@ node_modules/
 
 # Visual QA working output. Curated review sets are committed under docs/design.
 scripts/universe/visual-qa/artifacts/
+# Any output directory the harness is pointed at, not just the default one.
+scripts/universe/visual-qa/artifacts-*/
 scripts/universe/visual-qa/node_modules/

backend/docker-compose.test.yml

@@ -2,7 +2,10 @@ version: "3.7"
 
 services:
   db-test:
-    image: mariadb:10.5.21
+    # Must stay the same engine and major version the Universe deployment runs.
+    # The migrations use MariaDB syntax that MySQL rejects outright, so a test
+    # database on a different engine proves nothing about the release.
+    image: mariadb:11.4
     environment:
       MYSQL_DATABASE: "mempool_test"
       MYSQL_USER: "mempool_test"

backend/jest.integration.setup.ts

@@ -36,23 +36,49 @@ module.exports = async () => {
     const composeFile = path.join(__dirname, 'docker-compose.test.yml');
     const dockerComposeCmd = getDockerComposeCmd();
 
-    // Start the container
-    execSync(`${dockerComposeCmd} -f "${composeFile}" up -d`, {
+    // A container left running by an earlier run is not the container this run
+    // asked for: compose reports it as already up and never recreates it, so a
+    // change of image or of engine is silently ignored and the tests measure
+    // whatever was there before. Tear it down, volumes included, first.
+    try {
+      execSync(`${dockerComposeCmd} -f "${composeFile}" down -v --remove-orphans`, {
+        stdio: 'inherit',
+        cwd: __dirname
+      });
+    } catch {
+      // Nothing to remove on a clean machine, which is the normal case.
+    }
+
+    // `down` only removes what compose owns. A container left behind under the
+    // same name by anything else still holds the name, and `up` fails on the
+    // conflict rather than starting the database.
+    try {
+      execSync('docker rm -f backend-db-test-1', { stdio: 'pipe' });
+    } catch {
+      // No such container, which is the normal case.
+    }
+
+    execSync(`${dockerComposeCmd} -f "${composeFile}" up -d --force-recreate`, {
       stdio: 'inherit',
       cwd: __dirname
     });
 
     // Wait for database to be ready
     console.log('Waiting for database to be ready...');
     let attempts = 0;
-    const maxAttempts = 30;
+    const maxAttempts = 60;
 
     while (attempts < maxAttempts) {
       try {
-        execSync(`${dockerComposeCmd} -f "${composeFile}" exec -T db-test mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent`, {
-          cwd: __dirname,
-          stdio: 'pipe'
-        });
+        // MariaDB renamed its client binaries; either name may be the one this
+        // image ships, so a probe that only knows one of them reports a
+        // healthy server as a database that never started.
+        execSync(
+          `${dockerComposeCmd} -f "${composeFile}" exec -T db-test sh -c ` +
+            `"mariadb-admin ping -h localhost -u mempool_test -pmempool_test --silent ` +
+            `|| mysqladmin ping -h localhost -u mempool_test -pmempool_test --silent"`,
+          { cwd: __dirname, stdio: 'pipe' },
+        );
         console.log('Database is ready!');
         break;
       } catch (e) {

backend/mempool-config.sample.json

@@ -28,6 +28,7 @@
     "AUTOMATIC_POOLS_UPDATE": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "POOLS_JSON_FILE": "tasks/pools/pools-v2.json",
     "POOLS_UPDATE_DELAY": 604800,
     "AUDIT": false,
     "RUST_GBT": true,

backend/package.json

@@ -24,7 +24,7 @@
     "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
     "build": "npm run tsc && npm run create-resources",
     "clean": "rm -rf ./dist/ ./node_modules/ ./package/ ./rust-gbt/",
-    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "create-resources": "mkdir -p ./dist/tasks/pools && cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && cp ./src/tasks/pools/pools-v2.json ./dist/tasks/pools && node dist/api/fetch-version.js",
     "package": "./npm_package.sh",
     "package-rm-build-deps": "./npm_package_rm_build_deps.sh",
     "preinstall": "cd ../rust/gbt && npm run build-release && npm run to-backend",

backend/src/__fixtures__/mempool-config.template.json

@@ -29,6 +29,7 @@
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "POOLS_JSON_FILE": "__MEMPOOL_POOLS_JSON_FILE__",
     "POOLS_UPDATE_DELAY": 604800,
     "AUDIT": true,
     "RUST_GBT": false,

backend/src/__tests__/capabilities.test.ts

@@ -0,0 +1,61 @@
+import { preflightFailures, type PreflightInput } from '../api/capabilities.preflight';
+
+/**
+ * The production incident these rules exist to prevent: the frontend shipped a
+ * Charts page and a Mining dashboard while the backend had the database off, so
+ * every request behind those pages answered 404 and the pages loaded forever.
+ */
+
+const coherentWithData: PreflightInput = {
+  statisticsEnabled: true,
+  databaseEnabled: true,
+  mempoolEnabled: true,
+  indexingBlocksAmount: -1,
+};
+
+const coherentWithoutData: PreflightInput = {
+  statisticsEnabled: false,
+  databaseEnabled: false,
+  mempoolEnabled: true,
+  indexingBlocksAmount: 0,
+};
+
+describe('deployment configuration preflight', () => {
+  it('accepts a deployment that serves statistics and mining from a database', () => {
+    expect(preflightFailures(coherentWithData)).toEqual([]);
+  });
+
+  it('accepts a deployment that serves neither, with the database off', () => {
+    expect(preflightFailures(coherentWithoutData)).toEqual([]);
+  });
+
+  it('rejects statistics without a database, because the routes would never mount', () => {
+    const failures = preflightFailures({ ...coherentWithData, databaseEnabled: false, indexingBlocksAmount: 0 });
+    expect(failures.map((failure) => failure.feature)).toContain('statistics');
+  });
+
+  it('rejects statistics without the mempool backend collecting them', () => {
+    const failures = preflightFailures({ ...coherentWithData, mempoolEnabled: false });
+    expect(failures.map((failure) => failure.reason.includes('MEMPOOL.ENABLED'))).toContain(true);
+  });
+
+  it('rejects block indexing without a database, because mining routes would never mount', () => {
+    const failures = preflightFailures({ ...coherentWithoutData, indexingBlocksAmount: 52560 });
+    expect(failures.map((failure) => failure.feature)).toContain('mining');
+  });
+
+  it('rejects a database that nothing uses', () => {
+    const failures = preflightFailures({ ...coherentWithoutData, databaseEnabled: true });
+    expect(failures.map((failure) => failure.feature)).toContain('database');
+  });
+
+  it('reports every problem at once rather than stopping at the first', () => {
+    const failures = preflightFailures({
+      statisticsEnabled: true,
+      databaseEnabled: false,
+      mempoolEnabled: false,
+      indexingBlocksAmount: 144,
+    });
+    expect(failures.length).toBeGreaterThanOrEqual(3);
+  });
+});

backend/src/__tests__/config.test.ts

@@ -42,6 +42,7 @@ describe('Mempool Backend Config', () => {
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        POOLS_JSON_FILE: 'tasks/pools/pools-v2.json',
         POOLS_UPDATE_DELAY: 604800,
         AUDIT: false,
         RUST_GBT: true,

backend/src/api/capabilities.preflight.ts

@@ -0,0 +1,55 @@
+/**
+ * Configuration combinations that must never reach production traffic.
+ *
+ * This module deliberately imports nothing. The release gate, the unit tests,
+ * and the running backend all judge the same rules, and none of them has to
+ * open a database or start a timer to do it.
+ */
+
+/** A configuration combination that must never reach production traffic. */
+export interface PreflightFailure {
+  readonly feature: string;
+  readonly reason: string;
+}
+
+/** The configuration facts the preflight rules judge. */
+export interface PreflightInput {
+  readonly statisticsEnabled: boolean;
+  readonly databaseEnabled: boolean;
+  readonly mempoolEnabled: boolean;
+  readonly indexingBlocksAmount: number;
+}
+
+/**
+ * Returns every reason this configuration would put a broken public feature in
+ * front of users. An empty list means the deployment is coherent.
+ */
+export function preflightFailures(input: PreflightInput): PreflightFailure[] {
+  const failures: PreflightFailure[] = [];
+
+  if (input.statisticsEnabled && !input.databaseEnabled) {
+    failures.push({
+      feature: 'statistics',
+      reason: 'STATISTICS.ENABLED is true but DATABASE.ENABLED is false, so no statistics route can be served.',
+    });
+  }
+  if (input.statisticsEnabled && !input.mempoolEnabled) {
+    failures.push({
+      feature: 'statistics',
+      reason: 'STATISTICS.ENABLED is true but MEMPOOL.ENABLED is false, so nothing collects statistics.',
+    });
+  }
+  if (input.indexingBlocksAmount !== 0 && !input.databaseEnabled) {
+    failures.push({
+      feature: 'mining',
+      reason: 'INDEXING_BLOCKS_AMOUNT is set but DATABASE.ENABLED is false, so no mining route can be served.',
+    });
+  }
+  if (input.databaseEnabled && input.indexingBlocksAmount === 0 && !input.statisticsEnabled) {
+    failures.push({
+      feature: 'database',
+      reason: 'DATABASE.ENABLED is true but neither block indexing nor statistics uses it.',
+    });
+  }
+  return failures;
+}

backend/src/api/capabilities.routes.ts

@@ -0,0 +1,27 @@
+import { Application, Request, Response } from 'express';
+import config from '../config';
+import capabilities from './capabilities';
+import { handleError } from '../utils/api';
+
+/**
+ * Publishes what this deployment can actually serve. The frontend reads it to
+ * render truthful states instead of guessing from its own build-time flags,
+ * and the release procedure reads it to refuse an incoherent cutover.
+ */
+class CapabilitiesRoutes {
+  public initRoutes(app: Application): void {
+    app.get(config.MEMPOOL.API_URL_PREFIX + 'capabilities', async (req: Request, res: Response) => {
+      try {
+        const report = await capabilities.$report();
+        res.header('Pragma', 'public');
+        res.header('Cache-control', 'public');
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 10).toUTCString());
+        res.json(report);
+      } catch (e) {
+        handleError(req, res, 500, 'Failed to build the capability report');
+      }
+    });
+  }
+}
+
+export default new CapabilitiesRoutes();